diff --git a/.config/nextest.toml b/.config/nextest.toml
index 15de353595390..0e6cef6fd7388 100644
--- a/.config/nextest.toml
+++ b/.config/nextest.toml
@@ -34,7 +34,3 @@ test-group = 'failpoint-limited'
 [[profile.ci-sim.overrides]]
 filter = 'test(failpoint_limited::)'
 test-group = 'failpoint-limited'
-
-[[profile.ci.junit.overrides]]
-filter = 'test(failpoint_limited::)'
-test-group = 'failpoint-limited'
\ No newline at end of file
diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index b8ca322d767a8..c8e1f42656e47 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -42,3 +42,6 @@ c583e2c6c054764249acf484438c7bf7197765f4
 
 # chore: replace all ProstXxx with PbXxx (#8621)
 6fd8821f2e053957b183d648bea9c95b6703941f
+
+# chore: cleanup v2 naming for sql metastore (#18941)
+9a6a7f9052d5679165ff57cc01417c742c95351c
diff --git a/.github/labeler.yml b/.github/labeler.yml
index 5a87927411e04..4dbc6eaa80066 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -42,3 +42,7 @@ labels:
 - label: "ci/run-e2e-single-node-tests"
   files:
   - "src\\/meta\\/.*.rs"
+
+- label: "ci/run-e2e-test-other-backends"
+  files:
+  - "src\\/meta\\/.*.rs"
diff --git a/.gitignore b/.gitignore
index 5f54a467b21b7..81e8609f3334e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,7 +83,7 @@ e2e_test/iceberg/metastore_db
 **/*.sqlite
 **/*.sqlite-journal
 
-*.slt.temp
+*.slt*.temp
 
 .direnv/
 
diff --git a/Cargo.lock b/Cargo.lock
index a106b29e50dec..2e84e822497b4 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "addr2line"
@@ -405,10 +405,10 @@ version = "53.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03675e42d1560790f3524800e41403b40d0da1c793fe9528929fde06d8c7649a"
 dependencies = [
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
  "chrono",
  "half 2.3.1",
  "num",
@@ -449,14 +449,14 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cd2bf348cf9f02a5975c5962c7fa6dee107a2009a7b41ac5fb1a027e12dc033f"
+checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8"
 dependencies = [
  "ahash 0.8.11",
- "arrow-buffer 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-buffer 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
  "chrono",
  "half 2.3.1",
  "hashbrown 0.14.5",
@@ -487,9 +487,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3092e37715f168976012ce52273c3989b5793b0db5f06cbaa246be25e5f0924d"
+checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec"
 dependencies = [
  "bytes",
  "half 2.3.1",
@@ -510,7 +510,7 @@ dependencies = [
  "base64 0.21.7",
  "chrono",
  "half 2.3.1",
- "lexical-core",
+ "lexical-core 0.8.5",
  "num",
 ]
 
@@ -530,27 +530,27 @@ dependencies = [
  "chrono",
  "comfy-table",
  "half 2.3.1",
- "lexical-core",
+ "lexical-core 0.8.5",
  "num",
  "ryu",
 ]
 
 [[package]]
 name = "arrow-cast"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ce1018bb710d502f9db06af026ed3561552e493e989a79d0d0f5d9cf267a785"
+checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1"
 dependencies = [
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
- "arrow-select 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
+ "arrow-select 53.2.0",
  "atoi",
  "base64 0.22.0",
  "chrono",
  "half 2.3.1",
- "lexical-core",
+ "lexical-core 1.0.2",
  "num",
  "ryu",
 ]
@@ -570,7 +570,7 @@ dependencies = [
  "csv",
  "csv-core",
  "lazy_static",
- "lexical-core",
+ "lexical-core 0.8.5",
  "regex",
 ]
 
@@ -600,12 +600,12 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e4ac0c4ee79150afe067dc4857154b3ee9c1cd52b5f40d59a77306d0ed18d65"
+checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5"
 dependencies = [
- "arrow-buffer 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-buffer 53.2.0",
+ "arrow-schema 53.2.0",
  "half 2.3.1",
  "num",
 ]
@@ -616,11 +616,11 @@ version = "53.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b915fb36d935b969894d7909ad417c67ddeadebbbd57c3c168edf64721a37d31"
 dependencies = [
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-cast 53.0.0",
- "arrow-ipc 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-cast 53.2.0",
+ "arrow-ipc 53.2.0",
+ "arrow-schema 53.2.0",
  "base64 0.22.0",
  "bytes",
  "futures",
@@ -662,15 +662,15 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb307482348a1267f91b0912e962cd53440e5de0f7fb24c5f7b10da70b38c94a"
+checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee"
 dependencies = [
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-cast 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-cast 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
  "flatbuffers 24.3.25",
 ]
 
@@ -688,7 +688,7 @@ dependencies = [
  "chrono",
  "half 2.3.1",
  "indexmap 2.6.0",
- "lexical-core",
+ "lexical-core 0.8.5",
  "num",
  "serde",
  "serde_json",
@@ -730,11 +730,11 @@ version = "53.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "644046c479d80ae8ed02a7f1e1399072ea344ca6a7b0e293ab2d5d9ed924aa3b"
 dependencies = [
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
- "arrow-select 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
+ "arrow-select 53.2.0",
  "half 2.3.1",
  "num",
 ]
@@ -775,10 +775,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a29791f8eb13b340ce35525b723f5f0df17ecb955599e11f65c2a94ab34e2efb"
 dependencies = [
  "ahash 0.8.11",
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
  "half 2.3.1",
 ]
 
@@ -799,9 +799,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c85320a3a2facf2b2822b57aa9d6d9d55edb8aee0b6b5d3b8df158e503d10858"
+checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1"
 
 [[package]]
 name = "arrow-select"
@@ -833,15 +833,15 @@ dependencies = [
 
 [[package]]
 name = "arrow-select"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9cc7e6b582e23855fd1625ce46e51647aa440c20ea2e71b1d748e0839dd73cba"
+checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125"
 dependencies = [
  "ahash 0.8.11",
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
  "num",
 ]
 
@@ -884,11 +884,11 @@ version = "53.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0775b6567c66e56ded19b87a954b6b1beffbdd784ef95a3a2b03f59570c1d230"
 dependencies = [
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-data 53.0.0",
- "arrow-schema 53.0.0",
- "arrow-select 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-schema 53.2.0",
+ "arrow-select 53.2.0",
  "memchr",
  "num",
  "regex",
@@ -901,10 +901,10 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c9f5ae23e3833d68bc536f3e744802e2d87db31dc4adbb0541f73bc69cc789a5"
 dependencies = [
- "arrow-array 53.0.0",
+ "arrow-array 53.2.0",
  "arrow-flight",
- "arrow-schema 53.0.0",
- "arrow-select 53.0.0",
+ "arrow-schema 53.2.0",
+ "arrow-select 53.2.0",
  "futures-util",
  "thiserror",
  "tokio",
@@ -919,9 +919,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6151bb7f26cde846e14adb17e08282153f7a9250dd78bbab3fa462b66d7b623"
 dependencies = [
  "anyhow",
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-schema 53.2.0",
  "atomic-time",
  "rquickjs",
 ]
@@ -933,10 +933,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c0b80da061a53aac237e711fddb01709002ba2e006f9fd4c72a430d4938dd921"
 dependencies = [
  "anyhow",
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-ipc 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-ipc 53.2.0",
+ "arrow-schema 53.2.0",
  "pyo3",
  "pyo3-build-config",
 ]
@@ -948,9 +948,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fe31144804e093dd60b4e7a749b64b9454040c05a34ccbeb641fc60fcf5ee92d"
 dependencies = [
  "anyhow",
- "arrow-array 53.0.0",
- "arrow-ipc 53.0.0",
- "arrow-schema 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-ipc 53.2.0",
+ "arrow-schema 53.2.0",
  "async-trait",
  "base64 0.22.0",
  "genawaiter",
@@ -1040,7 +1040,7 @@ dependencies = [
  "async-channel 2.2.1",
  "async-executor",
  "async-io",
- "async-lock 3.4.0",
+ "async-lock",
  "blocking",
  "futures-lite",
  "once_cell",
@@ -1053,7 +1053,7 @@ version = "2.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0d6baa8f0178795da0e71bc42c9e5d13261aac7ee549853162e66a241ba17964"
 dependencies = [
- "async-lock 3.4.0",
+ "async-lock",
  "cfg-if",
  "concurrent-queue",
  "futures-io",
@@ -1066,15 +1066,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "async-lock"
-version = "2.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "287272293e9d8c41773cec55e365490fe034813a2f172f502d6ddcf75b2f582b"
-dependencies = [
- "event-listener 2.5.3",
-]
-
 [[package]]
 name = "async-lock"
 version = "3.4.0"
@@ -1141,7 +1132,7 @@ dependencies = [
  "async-channel 1.9.0",
  "async-global-executor",
  "async-io",
- "async-lock 3.4.0",
+ "async-lock",
  "crossbeam-utils",
  "futures-channel",
  "futures-core",
@@ -1822,7 +1813,7 @@ dependencies = [
  "serde_urlencoded",
  "sync_wrapper",
  "tokio",
- "tower",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -1880,7 +1871,7 @@ dependencies = [
  "pin-project-lite",
  "serde",
  "serde_html_form",
- "tower",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -1949,12 +1940,6 @@ version = "0.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf"
 
-[[package]]
-name = "base64"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
-
 [[package]]
 name = "base64"
 version = "0.13.1"
@@ -2108,11 +2093,11 @@ dependencies = [
 
 [[package]]
 name = "bit-set"
-version = "0.5.3"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
+checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3"
 dependencies = [
- "bit-vec",
+ "bit-vec 0.8.0",
 ]
 
 [[package]]
@@ -2121,6 +2106,12 @@ version = "0.6.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
 
+[[package]]
+name = "bit-vec"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7"
+
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -2246,6 +2237,17 @@ dependencies = [
  "brotli-decompressor",
 ]
 
+[[package]]
+name = "brotli"
+version = "7.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
 [[package]]
 name = "brotli-decompressor"
 version = "4.0.1"
@@ -2322,12 +2324,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "bytecount"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
-
 [[package]]
 name = "bytemuck"
 version = "1.14.0"
@@ -2389,15 +2385,6 @@ dependencies = [
  "pkg-config",
 ]
 
-[[package]]
-name = "camino"
-version = "1.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
-dependencies = [
- "serde",
-]
-
 [[package]]
 name = "cap-fs-ext"
 version = "3.0.0"
@@ -2469,28 +2456,6 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1582e1c9e755dd6ad6b224dcffb135d199399a4568d454bd89fe515ca8425695"
 
-[[package]]
-name = "cargo-platform"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc"
-dependencies = [
- "serde",
-]
-
-[[package]]
-name = "cargo_metadata"
-version = "0.14.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa"
-dependencies = [
- "camino",
- "cargo-platform",
- "semver 1.0.18",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "cast"
 version = "0.3.0"
@@ -2874,7 +2839,7 @@ dependencies = [
  "serde_json",
  "thread_local",
  "tokio",
- "tokio-stream 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-stream 0.1.16",
  "tonic",
  "tracing",
  "tracing-core",
@@ -3267,7 +3232,7 @@ dependencies = [
  "bitflags 1.3.2",
  "crossterm_winapi",
  "libc",
- "mio",
+ "mio 0.8.11",
  "parking_lot 0.12.1",
  "signal-hook",
  "signal-hook-mio",
@@ -4512,20 +4477,21 @@ dependencies = [
 
 [[package]]
 name = "elasticsearch"
-version = "8.5.0-alpha.1"
+version = "8.15.0-alpha.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "40d9bd57d914cc66ce878f098f63ed7b5d5b64c30644a5adb950b008f874a6c6"
+checksum = "d99147dd351d320432ec103a20804cbb593d3d08fd2eed0ee86f21a0f5137a3c"
 dependencies = [
- "base64 0.11.0",
+ "base64 0.22.0",
  "bytes",
  "dyn-clone",
  "lazy_static",
  "percent-encoding",
- "reqwest 0.11.20",
- "rustc_version 0.2.3",
+ "reqwest 0.12.4",
+ "rustc_version 0.4.0",
  "serde",
  "serde_json",
- "serde_with 1.14.0",
+ "serde_with 3.8.1",
+ "tokio",
  "url",
  "void",
 ]
@@ -4731,15 +4697,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "error-chain"
-version = "0.12.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc"
-dependencies = [
- "version_check",
-]
-
 [[package]]
 name = "escape8259"
 version = "0.5.2"
@@ -4831,9 +4788,9 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
 
 [[package]]
 name = "fancy-regex"
-version = "0.13.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2"
+checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298"
 dependencies = [
  "bit-set",
  "regex-automata 0.4.8",
@@ -5058,6 +5015,15 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "float-cmp"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8"
+dependencies = [
+ "num-traits",
+]
+
 [[package]]
 name = "flume"
 version = "0.11.0"
@@ -5536,7 +5502,7 @@ dependencies = [
  "thiserror",
  "time",
  "tokio",
- "tokio-stream 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-stream 0.1.16",
  "url",
  "yup-oauth2",
 ]
@@ -5627,7 +5593,7 @@ dependencies = [
  "thiserror",
  "tokio",
  "tonic",
- "tower",
+ "tower 0.4.13",
  "tracing",
  "trust-dns-resolver 0.23.2",
 ]
@@ -5711,7 +5677,7 @@ dependencies = [
  "tokio",
  "tokio-retry",
  "tonic",
- "tower",
+ "tower 0.4.13",
  "tracing",
 ]
 
@@ -5906,9 +5872,9 @@ checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb"
 
 [[package]]
 name = "hashlink"
-version = "0.8.4"
+version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
+checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
 dependencies = [
  "hashbrown 0.14.5",
 ]
@@ -5941,9 +5907,6 @@ name = "heck"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
-dependencies = [
- "unicode-segmentation",
-]
 
 [[package]]
 name = "heck"
@@ -6234,7 +6197,7 @@ dependencies = [
  "pin-project-lite",
  "socket2 0.5.6",
  "tokio",
- "tower",
+ "tower 0.4.13",
  "tower-service",
  "tracing",
 ]
@@ -6278,16 +6241,17 @@ dependencies = [
 [[package]]
 name = "iceberg"
 version = "0.3.0"
-source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=f8b7bff7bd7853be7733fc1db1203862878f92c1#f8b7bff7bd7853be7733fc1db1203862878f92c1"
+source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=e28726443a57028f7c7df11d6d385470dc484d46#e28726443a57028f7c7df11d6d385470dc484d46"
 dependencies = [
  "anyhow",
  "apache-avro 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "array-init",
  "arrow-arith 53.0.0",
- "arrow-array 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-cast 53.2.0",
  "arrow-ord 53.0.0",
- "arrow-schema 53.0.0",
- "arrow-select 53.0.0",
+ "arrow-schema 53.2.0",
+ "arrow-select 53.2.0",
  "arrow-string 53.0.0",
  "async-trait",
  "bimap",
@@ -6300,10 +6264,11 @@ dependencies = [
  "itertools 0.13.0",
  "moka",
  "murmur3",
+ "num-bigint",
  "once_cell",
- "opendal 0.50.0",
+ "opendal 0.50.1",
  "ordered-float 4.1.1",
- "parquet 53.0.0",
+ "parquet 53.2.0",
  "paste",
  "rand",
  "reqwest 0.12.4",
@@ -6323,7 +6288,7 @@ dependencies = [
 [[package]]
 name = "iceberg-catalog-glue"
 version = "0.3.0"
-source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=f8b7bff7bd7853be7733fc1db1203862878f92c1#f8b7bff7bd7853be7733fc1db1203862878f92c1"
+source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=e28726443a57028f7c7df11d6d385470dc484d46#e28726443a57028f7c7df11d6d385470dc484d46"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -6340,7 +6305,7 @@ dependencies = [
 [[package]]
 name = "iceberg-catalog-rest"
 version = "0.3.0"
-source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=f8b7bff7bd7853be7733fc1db1203862878f92c1#f8b7bff7bd7853be7733fc1db1203862878f92c1"
+source = "git+https://github.com/risingwavelabs/iceberg-rust.git?rev=e28726443a57028f7c7df11d6d385470dc484d46#e28726443a57028f7c7df11d6d385470dc484d46"
 dependencies = [
  "async-trait",
  "chrono",
@@ -6365,13 +6330,13 @@ dependencies = [
  "anyhow",
  "apache-avro 0.17.0 (git+https://github.com/apache/avro.git)",
  "arrow-arith 53.0.0",
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-cast 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-cast 53.2.0",
  "arrow-ord 53.0.0",
  "arrow-row 53.0.0",
- "arrow-schema 53.0.0",
- "arrow-select 53.0.0",
+ "arrow-schema 53.2.0",
+ "arrow-select 53.2.0",
  "async-trait",
  "bitvec",
  "bytes",
@@ -6388,7 +6353,7 @@ dependencies = [
  "once_cell",
  "opendal 0.49.2",
  "ordered-float 3.9.1",
- "parquet 53.0.0",
+ "parquet 53.2.0",
  "prometheus",
  "regex",
  "reqwest 0.11.20",
@@ -6525,9 +6490,9 @@ dependencies = [
 
 [[package]]
 name = "inotify"
-version = "0.9.6"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8069d3ec154eb856955c1c0fbffefbf5f3c40a104ec912d4797314c1801abff"
+checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc"
 dependencies = [
  "bitflags 1.3.2",
  "inotify-sys",
@@ -6890,11 +6855,24 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46"
 dependencies = [
- "lexical-parse-float",
- "lexical-parse-integer",
- "lexical-util",
- "lexical-write-float",
- "lexical-write-integer",
+ "lexical-parse-float 0.8.5",
+ "lexical-parse-integer 0.8.6",
+ "lexical-util 0.8.5",
+ "lexical-write-float 0.8.5",
+ "lexical-write-integer 0.8.5",
+]
+
+[[package]]
+name = "lexical-core"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458"
+dependencies = [
+ "lexical-parse-float 1.0.2",
+ "lexical-parse-integer 1.0.2",
+ "lexical-util 1.0.3",
+ "lexical-write-float 1.0.2",
+ "lexical-write-integer 1.0.2",
 ]
 
 [[package]]
@@ -6903,8 +6881,19 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f"
 dependencies = [
- "lexical-parse-integer",
- "lexical-util",
+ "lexical-parse-integer 0.8.6",
+ "lexical-util 0.8.5",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-parse-float"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0"
+dependencies = [
+ "lexical-parse-integer 1.0.2",
+ "lexical-util 1.0.3",
  "static_assertions",
 ]
 
@@ -6914,7 +6903,17 @@ version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6d0994485ed0c312f6d965766754ea177d07f9c00c9b82a5ee62ed5b47945ee9"
 dependencies = [
- "lexical-util",
+ "lexical-util 0.8.5",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-parse-integer"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61"
+dependencies = [
+ "lexical-util 1.0.3",
  "static_assertions",
 ]
 
@@ -6927,14 +6926,34 @@ dependencies = [
  "static_assertions",
 ]
 
+[[package]]
+name = "lexical-util"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0"
+dependencies = [
+ "static_assertions",
+]
+
 [[package]]
 name = "lexical-write-float"
 version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862"
 dependencies = [
- "lexical-util",
- "lexical-write-integer",
+ "lexical-util 0.8.5",
+ "lexical-write-integer 0.8.5",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-write-float"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809"
+dependencies = [
+ "lexical-util 1.0.3",
+ "lexical-write-integer 1.0.2",
  "static_assertions",
 ]
 
@@ -6944,7 +6963,17 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446"
 dependencies = [
- "lexical-util",
+ "lexical-util 0.8.5",
+ "static_assertions",
+]
+
+[[package]]
+name = "lexical-write-integer"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162"
+dependencies = [
+ "lexical-util 1.0.3",
  "static_assertions",
 ]
 
@@ -7018,9 +7047,9 @@ dependencies = [
 
 [[package]]
 name = "libsqlite3-sys"
-version = "0.27.0"
+version = "0.30.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
+checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
 dependencies = [
  "cc",
  "pkg-config",
@@ -7236,8 +7265,9 @@ dependencies = [
 
 [[package]]
 name = "madsim"
-version = "0.2.30"
-source = "git+https://github.com/madsim-rs/madsim.git?rev=f0ffd24864245aea236833f1ff3638f75bd51700#f0ffd24864245aea236833f1ff3638f75bd51700"
+version = "0.2.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3c97f34bb19cf6a435a4da2187e90acc6bc59faa730e493b28b6d33e1bb9ccb"
 dependencies = [
  "ahash 0.8.11",
  "async-channel 2.2.1",
@@ -7285,7 +7315,8 @@ dependencies = [
 [[package]]
 name = "madsim-macros"
 version = "0.2.12"
-source = "git+https://github.com/madsim-rs/madsim.git?rev=f0ffd24864245aea236833f1ff3638f75bd51700#f0ffd24864245aea236833f1ff3638f75bd51700"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1"
 dependencies = [
  "darling 0.14.4",
  "proc-macro2",
@@ -7340,7 +7371,7 @@ dependencies = [
  "madsim",
  "tokio",
  "tonic",
- "tower",
+ "tower 0.4.13",
  "tracing",
 ]
 
@@ -7573,6 +7604,19 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "mio"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec"
+dependencies = [
+ "hermit-abi 0.3.9",
+ "libc",
+ "log",
+ "wasi",
+ "windows-sys 0.52.0",
+]
+
 [[package]]
 name = "mirai-annotations"
 version = "1.12.0"
@@ -7608,21 +7652,21 @@ dependencies = [
 
 [[package]]
 name = "moka"
-version = "0.12.0"
+version = "0.12.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8dc65d4615c08c8a13d91fd404b5a2a4485ba35b4091e3315cf8798d280c2f29"
+checksum = "32cf62eb4dd975d2dde76432fb1075c49e3ee2331cf36f1f8fd4b66550d32b6f"
 dependencies = [
- "async-lock 2.8.0",
+ "async-lock",
  "async-trait",
  "crossbeam-channel",
  "crossbeam-epoch",
  "crossbeam-utils",
+ "event-listener 5.3.1",
  "futures-util",
  "once_cell",
  "parking_lot 0.12.1",
  "quanta",
  "rustc_version 0.4.0",
- "skeptic",
  "smallvec",
  "tagptr",
  "thiserror",
@@ -7737,7 +7781,7 @@ dependencies = [
  "keyed_priority_queue",
  "lazy_static",
  "lru 0.12.0",
- "mio",
+ "mio 0.8.11",
  "mysql_common",
  "native-tls",
  "once_cell",
@@ -7918,9 +7962,9 @@ checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
 
 [[package]]
 name = "notify"
-version = "6.1.1"
+version = "7.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6205bd8bb1e454ad2e27422015fb5e4f2bcc7e08fa8f27058670d208324a4d2d"
+checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009"
 dependencies = [
  "bitflags 2.6.0",
  "filetime",
@@ -7929,9 +7973,19 @@ dependencies = [
  "kqueue",
  "libc",
  "log",
- "mio",
+ "mio 1.0.2",
+ "notify-types",
  "walkdir",
- "windows-sys 0.48.0",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "notify-types"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7393c226621f817964ffb3dc5704f9509e107a8b024b489cc2c1b217378785df"
+dependencies = [
+ "instant",
 ]
 
 [[package]]
@@ -8250,9 +8304,9 @@ dependencies = [
 
 [[package]]
 name = "opendal"
-version = "0.50.0"
+version = "0.50.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36e44fc43be9ffe18dad3e3ef9d61c1ae01991ee6f1c8c026978c35777a711bf"
+checksum = "213222b6c86949314d8f51acb26d8241e7c8dd0879b016a79471d49f21ee592f"
 dependencies = [
  "anyhow",
  "async-trait",
@@ -8452,7 +8506,7 @@ dependencies = [
  "serde_json",
  "thiserror",
  "tokio",
- "tokio-stream 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-stream 0.1.16",
 ]
 
 [[package]]
@@ -8533,9 +8587,9 @@ dependencies = [
 
 [[package]]
 name = "ouroboros"
-version = "0.17.2"
+version = "0.18.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e2ba07320d39dfea882faa70554b4bd342a5f273ed59ba7c1c6b4c840492c954"
+checksum = "944fa20996a25aded6b4795c6d63f10014a7a83f8be9828a11860b08c5fc4a67"
 dependencies = [
  "aliasable",
  "ouroboros_macro",
@@ -8544,13 +8598,14 @@ dependencies = [
 
 [[package]]
 name = "ouroboros_macro"
-version = "0.17.2"
+version = "0.18.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ec4c6225c69b4ca778c0aea097321a64c421cf4577b331c61b229267edabb6f8"
+checksum = "39b0deead1528fd0e5947a8546a9642a9777c25f6e1e26f34c97b204bbb465bd"
 dependencies = [
  "heck 0.4.1",
- "proc-macro-error 1.0.4",
+ "itertools 0.12.1",
  "proc-macro2",
+ "proc-macro2-diagnostics",
  "quote",
  "syn 2.0.79",
 ]
@@ -8686,7 +8741,7 @@ dependencies = [
  "arrow-schema 52.2.0",
  "arrow-select 52.2.0",
  "base64 0.22.0",
- "brotli",
+ "brotli 6.0.0",
  "bytes",
  "chrono",
  "flate2",
@@ -8709,20 +8764,20 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "53.0.0"
+version = "53.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0fbf928021131daaa57d334ca8e3904fe9ae22f73c56244fc7db9b04eedc3d8"
+checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e"
 dependencies = [
  "ahash 0.8.11",
- "arrow-array 53.0.0",
- "arrow-buffer 53.0.0",
- "arrow-cast 53.0.0",
- "arrow-data 53.0.0",
- "arrow-ipc 53.0.0",
- "arrow-schema 53.0.0",
- "arrow-select 53.0.0",
+ "arrow-array 53.2.0",
+ "arrow-buffer 53.2.0",
+ "arrow-cast 53.2.0",
+ "arrow-data 53.2.0",
+ "arrow-ipc 53.2.0",
+ "arrow-schema 53.2.0",
+ "arrow-select 53.2.0",
  "base64 0.22.0",
- "brotli",
+ "brotli 7.0.0",
  "bytes",
  "chrono",
  "flate2",
@@ -9239,7 +9294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd"
 dependencies = [
  "difflib",
- "float-cmp",
+ "float-cmp 0.9.0",
  "itertools 0.10.5",
  "normalize-line-endings",
  "predicates-core",
@@ -9287,7 +9342,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66"
 dependencies = [
  "diff",
- "yansi",
+ "yansi 0.5.1",
 ]
 
 [[package]]
@@ -9403,6 +9458,19 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "proc-macro2-diagnostics"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.79",
+ "version_check",
+ "yansi 1.0.1",
+]
+
 [[package]]
 name = "procfs"
 version = "0.14.2"
@@ -9686,17 +9754,6 @@ dependencies = [
  "syn 1.0.109",
 ]
 
-[[package]]
-name = "pulldown-cmark"
-version = "0.9.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57206b407293d2bcd3af849ce869d52068623f19e1b5ff8e8778e3309439682b"
-dependencies = [
- "bitflags 2.6.0",
- "memchr",
- "unicase",
-]
-
 [[package]]
 name = "pulsar"
 version = "6.3.0"
@@ -9704,7 +9761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d7f3541ff84e39da334979ac4bf171e0f277f4f782603aeae65bf5795dc7275a"
 dependencies = [
  "async-trait",
- "bit-vec",
+ "bit-vec 0.6.3",
  "bytes",
  "chrono",
  "crc",
@@ -9806,12 +9863,11 @@ checksum = "658fa1faf7a4cc5f057c9ee5ef560f717ad9d8dc66d975267f709624d6e1ab88"
 
 [[package]]
 name = "quanta"
-version = "0.11.0"
-source = "git+https://github.com/madsim-rs/quanta.git?rev=948bdc3#948bdc3d4cd3fcfe3d52d03dd83deee96d97d770"
+version = "0.12.3"
+source = "git+https://github.com/madsim-rs/quanta.git?rev=ea9ba802327b1d72c4b1c7202c759b0a5243271e#ea9ba802327b1d72c4b1c7202c759b0a5243271e"
 dependencies = [
  "crossbeam-utils",
  "libc",
- "mach2",
  "once_cell",
  "raw-cpuid",
  "wasi",
@@ -9910,11 +9966,11 @@ dependencies = [
 
 [[package]]
 name = "raw-cpuid"
-version = "10.7.0"
+version = "11.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
+checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0"
 dependencies = [
- "bitflags 1.3.2",
+ "bitflags 2.6.0",
 ]
 
 [[package]]
@@ -10163,7 +10219,6 @@ version = "0.11.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
 dependencies = [
- "async-compression",
  "base64 0.21.7",
  "bytes",
  "encoding_rs",
@@ -10191,7 +10246,6 @@ dependencies = [
  "tokio",
  "tokio-native-tls",
  "tokio-rustls 0.24.1",
- "tokio-util",
  "tower-service",
  "url",
  "wasm-bindgen",
@@ -10430,6 +10484,7 @@ dependencies = [
  "async-recursion",
  "async-trait",
  "bytes",
+ "chrono",
  "criterion",
  "either",
  "foyer",
@@ -10443,9 +10498,10 @@ dependencies = [
  "madsim-tokio",
  "madsim-tonic",
  "memcomparable",
+ "mysql_async",
  "opendal 0.49.2",
  "parking_lot 0.12.1",
- "parquet 53.0.0",
+ "parquet 53.2.0",
  "paste",
  "prometheus",
  "prost 0.13.1",
@@ -10460,6 +10516,7 @@ dependencies = [
  "risingwave_pb",
  "risingwave_rpc_client",
  "risingwave_storage",
+ "rust_decimal",
  "rw_futures_util",
  "scopeguard",
  "serde_json",
@@ -10469,7 +10526,7 @@ dependencies = [
  "tikv-jemallocator",
  "tokio-metrics",
  "tokio-postgres",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "tokio-util",
  "tracing",
  "twox-hash",
@@ -10512,7 +10569,7 @@ dependencies = [
  "serde",
  "serde_yaml",
  "thiserror-ext",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "toml 0.8.12",
  "tracing",
  "tracing-subscriber",
@@ -10580,13 +10637,13 @@ dependencies = [
  "anyhow",
  "arc-swap",
  "arrow-array 52.2.0",
- "arrow-array 53.0.0",
+ "arrow-array 53.2.0",
  "arrow-buffer 52.2.0",
- "arrow-buffer 53.0.0",
+ "arrow-buffer 53.2.0",
  "arrow-cast 52.2.0",
- "arrow-cast 53.0.0",
+ "arrow-cast 53.2.0",
  "arrow-schema 52.2.0",
- "arrow-schema 53.0.0",
+ "arrow-schema 53.2.0",
  "async-trait",
  "auto_enums",
  "auto_impl",
@@ -10775,6 +10832,7 @@ dependencies = [
  "aes-gcm",
  "anyhow",
  "bincode 1.3.3",
+ "hex",
  "parking_lot 0.12.1",
  "prost 0.13.1",
  "risingwave_pb",
@@ -10802,7 +10860,7 @@ dependencies = [
  "serde",
  "thiserror",
  "thiserror-ext",
- "tower",
+ "tower 0.5.0",
  "tower-http",
  "tracing",
  "workspace-hack",
@@ -10900,8 +10958,8 @@ dependencies = [
  "tempfile",
  "thiserror-ext",
  "tikv-jemalloc-ctl",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
- "tower",
+ "tokio-stream 0.1.15",
+ "tower 0.5.0",
  "tracing",
  "uuid",
  "workspace-hack",
@@ -10979,7 +11037,7 @@ dependencies = [
  "opensearch",
  "openssl",
  "parking_lot 0.12.1",
- "parquet 53.0.0",
+ "parquet 53.2.0",
  "paste",
  "pg_bigdecimal",
  "postgres-openssl",
@@ -11008,7 +11066,7 @@ dependencies = [
  "rustls-pemfile 2.2.0",
  "rustls-pki-types",
  "rw_futures_util",
- "sea-schema 0.15.0",
+ "sea-schema",
  "serde",
  "serde_derive",
  "serde_json",
@@ -11026,7 +11084,7 @@ dependencies = [
  "time",
  "tokio-postgres",
  "tokio-retry",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "tokio-util",
  "tracing",
  "tracing-subscriber",
@@ -11312,6 +11370,7 @@ dependencies = [
  "maplit",
  "md5",
  "memcomparable",
+ "mysql_async",
  "num-integer",
  "parking_lot 0.12.1",
  "parse-display",
@@ -11351,7 +11410,7 @@ dependencies = [
  "thiserror",
  "thiserror-ext",
  "tokio-postgres",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "tracing",
  "uuid",
  "workspace-hack",
@@ -11562,6 +11621,7 @@ dependencies = [
  "maplit",
  "memcomparable",
  "mime_guess",
+ "moka",
  "notify",
  "num-integer",
  "num-traits",
@@ -11597,8 +11657,8 @@ dependencies = [
  "thiserror",
  "thiserror-ext",
  "tokio-retry",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
- "tower",
+ "tokio-stream 0.1.15",
+ "tower 0.5.0",
  "tower-http",
  "tracing",
  "tracing-subscriber",
@@ -11646,6 +11706,7 @@ name = "risingwave_meta_model_migration"
 version = "2.2.0-alpha"
 dependencies = [
  "async-std",
+ "easy-ext",
  "sea-orm",
  "sea-orm-migration",
  "serde",
@@ -11710,7 +11771,7 @@ dependencies = [
  "serde_json",
  "sync-point",
  "thiserror-ext",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "tracing",
  "workspace-hack",
 ]
@@ -11744,7 +11805,6 @@ dependencies = [
  "reqwest 0.12.4",
  "risingwave_common",
  "risingwave_jni_core",
- "rustls 0.23.5",
  "spin 0.9.8",
  "thiserror",
  "thiserror-ext",
@@ -11836,8 +11896,8 @@ dependencies = [
  "thiserror",
  "thiserror-ext",
  "tokio-retry",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
- "tower",
+ "tokio-stream 0.1.15",
+ "tower 0.5.0",
  "tracing",
  "url",
  "workspace-hack",
@@ -11865,6 +11925,7 @@ dependencies = [
  "risingwave_common",
  "risingwave_variables",
  "rlimit",
+ "rustls 0.23.5",
  "thiserror-ext",
  "time",
  "tokio",
@@ -11922,7 +11983,7 @@ dependencies = [
  "tempfile",
  "tikv-jemallocator",
  "tokio-postgres",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "tracing",
  "tracing-subscriber",
  "uuid",
@@ -11991,7 +12052,7 @@ dependencies = [
  "serde",
  "serde_with 3.8.1",
  "tokio-postgres",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "toml 0.8.12",
  "tracing",
  "workspace-hack",
@@ -12132,7 +12193,7 @@ dependencies = [
  "thiserror-ext",
  "tokio-metrics",
  "tokio-retry",
- "tokio-stream 0.1.15 (git+https://github.com/madsim-rs/tokio.git?rev=0dd1055)",
+ "tokio-stream 0.1.15",
  "tracing",
  "tracing-test",
  "workspace-hack",
@@ -12732,25 +12793,25 @@ dependencies = [
 
 [[package]]
 name = "sea-orm"
-version = "0.12.15"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c8814e37dc25de54398ee62228323657520b7f29713b8e238649385dbe473ee0"
+checksum = "d5680a8b686985116607ef5f5af2b1f9e1cc2c228330e93101816a0baa279afa"
 dependencies = [
  "async-stream",
  "async-trait",
- "bigdecimal 0.3.1",
+ "bigdecimal 0.4.5",
  "chrono",
  "futures",
  "log",
  "ouroboros",
  "rust_decimal",
  "sea-orm-macros",
- "sea-query 0.30.7",
- "sea-query-binder 0.5.0",
+ "sea-query",
+ "sea-query-binder",
  "serde",
  "serde_json",
  "sqlx",
- "strum 0.25.0",
+ "strum 0.26.3",
  "thiserror",
  "time",
  "tracing",
@@ -12760,16 +12821,16 @@ dependencies = [
 
 [[package]]
 name = "sea-orm-cli"
-version = "0.12.14"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "465ea2308d4716837e9af4a2cff8e14c28135867a580bb93e9e03d408a3a6afb"
+checksum = "70a157f42d291ccbd6e913b9d9b12dbe2ccbcf0472efc60c8715dd1254083aec"
 dependencies = [
  "chrono",
  "clap",
  "dotenvy",
  "glob",
  "regex",
- "sea-schema 0.14.2",
+ "sea-schema",
  "tracing",
  "tracing-subscriber",
  "url",
@@ -12777,9 +12838,9 @@ dependencies = [
 
 [[package]]
 name = "sea-orm-macros"
-version = "0.12.15"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5e115c6b078e013aa963cc2d38c196c2c40b05f03d0ac872fe06b6e0d5265603"
+checksum = "3a239e3bb1b566ad4ec2654d0d193d6ceddfd733487edc9c21a64d214c773910"
 dependencies = [
  "heck 0.4.1",
  "proc-macro2",
@@ -12791,9 +12852,9 @@ dependencies = [
 
 [[package]]
 name = "sea-orm-migration"
-version = "0.12.14"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac734b6e5610c2764056cc8495fbc293cd1c8ebe084fdfb74c3b0cdaaff9bb92"
+checksum = "63ba07e9f2479cc671758fcb1edee42ff2e32c34b3e67ab41d0af1e41f73c74e"
 dependencies = [
  "async-trait",
  "clap",
@@ -12801,20 +12862,19 @@ dependencies = [
  "futures",
  "sea-orm",
  "sea-orm-cli",
- "sea-schema 0.14.2",
+ "sea-schema",
  "tracing",
  "tracing-subscriber",
 ]
 
 [[package]]
 name = "sea-query"
-version = "0.30.7"
+version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4166a1e072292d46dc91f31617c2a1cdaf55a8be4b5c9f4bf2ba248e3ac4999b"
+checksum = "ff504d13b5e4b52fffcf2fb203d0352a5722fa5151696db768933e41e1e591bb"
 dependencies = [
- "bigdecimal 0.3.1",
+ "bigdecimal 0.4.5",
  "chrono",
- "derivative",
  "inherent",
  "ordered-float 3.9.1",
  "rust_decimal",
@@ -12824,91 +12884,49 @@ dependencies = [
  "uuid",
 ]
 
-[[package]]
-name = "sea-query"
-version = "0.31.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e5073b2cfed767511a57d18115f3b3d8bcb5690bf8c89518caec6cb22c0cd74"
-dependencies = [
- "inherent",
- "sea-query-derive",
-]
-
 [[package]]
 name = "sea-query-binder"
-version = "0.5.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36bbb68df92e820e4d5aeb17b4acd5cc8b5d18b2c36a4dd6f4626aabfa7ab1b9"
+checksum = "b0019f47430f7995af63deda77e238c17323359af241233ec768aba1faea7608"
 dependencies = [
- "bigdecimal 0.3.1",
+ "bigdecimal 0.4.5",
  "chrono",
  "rust_decimal",
- "sea-query 0.30.7",
+ "sea-query",
  "serde_json",
  "sqlx",
  "time",
  "uuid",
 ]
 
-[[package]]
-name = "sea-query-binder"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "754965d4aee6145bec25d0898e5c931e6c22859789ce62fd85a42a15ed5a8ce3"
-dependencies = [
- "sea-query 0.31.0",
- "sqlx",
-]
-
 [[package]]
 name = "sea-query-derive"
-version = "0.4.0"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd78f2e0ee8e537e9195d1049b752e0433e2cac125426bccb7b5c3e508096117"
+checksum = "9834af2c4bd8c5162f00c89f1701fb6886119a88062cf76fe842ea9e232b9839"
 dependencies = [
+ "darling 0.20.9",
  "heck 0.4.1",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.79",
  "thiserror",
 ]
 
 [[package]]
 name = "sea-schema"
-version = "0.14.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30d148608012d25222442d1ebbfafd1228dbc5221baf4ec35596494e27a2394e"
-dependencies = [
- "futures",
- "sea-query 0.30.7",
- "sea-schema-derive 0.2.0",
-]
-
-[[package]]
-name = "sea-schema"
-version = "0.15.0"
+version = "0.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad52149fc81836ea7424c3425d8f6ed8ad448dd16d2e4f6a3907ba46f3f2fd78"
+checksum = "aab1592d17860a9a8584d9b549aebcd06f7bdc3ff615f71752486ba0b05b1e6e"
 dependencies = [
  "futures",
- "sea-query 0.31.0",
- "sea-query-binder 0.6.0",
- "sea-schema-derive 0.3.0",
+ "sea-query",
+ "sea-query-binder",
+ "sea-schema-derive",
  "sqlx",
 ]
 
-[[package]]
-name = "sea-schema-derive"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6f686050f76bffc4f635cda8aea6df5548666b830b52387e8bc7de11056d11e"
-dependencies = [
- "heck 0.4.1",
- "proc-macro2",
- "quote",
- "syn 1.0.109",
-]
-
 [[package]]
 name = "sea-schema-derive"
 version = "0.3.0"
@@ -13022,9 +13040,6 @@ name = "semver"
 version = "1.0.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918"
-dependencies = [
- "serde",
-]
 
 [[package]]
 name = "semver-parser"
@@ -13389,7 +13404,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af"
 dependencies = [
  "libc",
- "mio",
+ "mio 0.8.11",
  "signal-hook",
 ]
 
@@ -13436,13 +13451,12 @@ dependencies = [
 
 [[package]]
 name = "simd-json"
-version = "0.13.3"
+version = "0.14.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d123f285a3635e423ec2ef5b67e0168dcf86c0d62fffbcea88fcd1c926e47413"
+checksum = "b1df0290e9bfe79ddd5ff8798ca887cd107b75353d2957efe9777296e17f26b5"
 dependencies = [
  "getrandom",
  "halfbrown",
- "lexical-core",
  "ref-cast",
  "serde",
  "serde_json",
@@ -13486,21 +13500,6 @@ version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9fed904c7fb2856d868b92464fc8fa597fce366edea1a9cbfaa8cb5fe080bd6d"
 
-[[package]]
-name = "skeptic"
-version = "0.13.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "16d23b015676c90a0f01c197bfdc786c20342c73a0afdda9025adb0bc42940a8"
-dependencies = [
- "bytecount",
- "cargo_metadata",
- "error-chain",
- "glob",
- "pulldown-cmark",
- "tempfile",
- "walkdir",
-]
-
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -13600,12 +13599,12 @@ dependencies = [
 
 [[package]]
 name = "speedate"
-version = "0.14.0"
+version = "0.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c323c4e6fece5a5a1a2a7f726d243144cce9fbcfe3ce4d9f3c6ede726a2bc780"
+checksum = "9a5e7adf4e07e7de39a64d77962ca14a09165e592d42d0c9f9acadb679f4f937"
 dependencies = [
- "strum 0.25.0",
- "strum_macros 0.25.3",
+ "strum 0.26.3",
+ "strum_macros 0.26.4",
 ]
 
 [[package]]
@@ -13729,8 +13728,8 @@ dependencies = [
 
 [[package]]
 name = "sqlx"
-version = "0.7.4"
-source = "git+https://github.com/kwannoel/sqlx.git?rev=ddf222f56cf99f865231a5383053645c6ea05ba3#ddf222f56cf99f865231a5383053645c6ea05ba3"
+version = "0.8.2"
+source = "git+https://github.com/madsim-rs/sqlx.git?rev=3efe6d0065963db2a2b7f30dee69f647e28dec81#3efe6d0065963db2a2b7f30dee69f647e28dec81"
 dependencies = [
  "sqlx-core",
  "sqlx-macros",
@@ -13741,24 +13740,24 @@ dependencies = [
 
 [[package]]
 name = "sqlx-core"
-version = "0.7.4"
-source = "git+https://github.com/kwannoel/sqlx.git?rev=ddf222f56cf99f865231a5383053645c6ea05ba3#ddf222f56cf99f865231a5383053645c6ea05ba3"
+version = "0.8.2"
+source = "git+https://github.com/madsim-rs/sqlx.git?rev=3efe6d0065963db2a2b7f30dee69f647e28dec81#3efe6d0065963db2a2b7f30dee69f647e28dec81"
 dependencies = [
- "ahash 0.8.11",
  "atoi",
- "bigdecimal 0.3.1",
+ "bigdecimal 0.4.5",
  "byteorder",
  "bytes",
  "chrono",
  "crc",
  "crossbeam-queue",
  "either",
- "event-listener 2.5.3",
+ "event-listener 5.3.1",
  "futures-channel",
  "futures-core",
  "futures-intrusive",
  "futures-io",
  "futures-util",
+ "hashbrown 0.14.5",
  "hashlink",
  "hex",
  "indexmap 2.6.0",
@@ -13777,7 +13776,7 @@ dependencies = [
  "sqlformat",
  "thiserror",
  "time",
- "tokio-stream 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-stream 0.1.16",
  "tracing",
  "url",
  "uuid",
@@ -13785,24 +13784,24 @@ dependencies = [
 
 [[package]]
 name = "sqlx-macros"
-version = "0.7.4"
-source = "git+https://github.com/kwannoel/sqlx.git?rev=ddf222f56cf99f865231a5383053645c6ea05ba3#ddf222f56cf99f865231a5383053645c6ea05ba3"
+version = "0.8.2"
+source = "git+https://github.com/madsim-rs/sqlx.git?rev=3efe6d0065963db2a2b7f30dee69f647e28dec81#3efe6d0065963db2a2b7f30dee69f647e28dec81"
 dependencies = [
  "proc-macro2",
  "quote",
  "sqlx-core",
  "sqlx-macros-core",
- "syn 1.0.109",
+ "syn 2.0.79",
 ]
 
 [[package]]
 name = "sqlx-macros-core"
-version = "0.7.4"
-source = "git+https://github.com/kwannoel/sqlx.git?rev=ddf222f56cf99f865231a5383053645c6ea05ba3#ddf222f56cf99f865231a5383053645c6ea05ba3"
+version = "0.8.2"
+source = "git+https://github.com/madsim-rs/sqlx.git?rev=3efe6d0065963db2a2b7f30dee69f647e28dec81#3efe6d0065963db2a2b7f30dee69f647e28dec81"
 dependencies = [
  "dotenvy",
  "either",
- "heck 0.4.1",
+ "heck 0.5.0",
  "hex",
  "madsim-tokio",
  "once_cell",
@@ -13815,19 +13814,19 @@ dependencies = [
  "sqlx-mysql",
  "sqlx-postgres",
  "sqlx-sqlite",
- "syn 1.0.109",
+ "syn 2.0.79",
  "tempfile",
  "url",
 ]
 
 [[package]]
 name = "sqlx-mysql"
-version = "0.7.4"
-source = "git+https://github.com/kwannoel/sqlx.git?rev=ddf222f56cf99f865231a5383053645c6ea05ba3#ddf222f56cf99f865231a5383053645c6ea05ba3"
+version = "0.8.2"
+source = "git+https://github.com/madsim-rs/sqlx.git?rev=3efe6d0065963db2a2b7f30dee69f647e28dec81#3efe6d0065963db2a2b7f30dee69f647e28dec81"
 dependencies = [
  "atoi",
- "base64 0.21.7",
- "bigdecimal 0.3.1",
+ "base64 0.22.0",
+ "bigdecimal 0.4.5",
  "bitflags 2.6.0",
  "byteorder",
  "bytes",
@@ -13868,12 +13867,12 @@ dependencies = [
 
 [[package]]
 name = "sqlx-postgres"
-version = "0.7.4"
-source = "git+https://github.com/kwannoel/sqlx.git?rev=ddf222f56cf99f865231a5383053645c6ea05ba3#ddf222f56cf99f865231a5383053645c6ea05ba3"
+version = "0.8.2"
+source = "git+https://github.com/madsim-rs/sqlx.git?rev=3efe6d0065963db2a2b7f30dee69f647e28dec81#3efe6d0065963db2a2b7f30dee69f647e28dec81"
 dependencies = [
  "atoi",
- "base64 0.21.7",
- "bigdecimal 0.3.1",
+ "base64 0.22.0",
+ "bigdecimal 0.4.5",
  "bitflags 2.6.0",
  "byteorder",
  "chrono",
@@ -13911,10 +13910,9 @@ dependencies = [
 
 [[package]]
 name = "sqlx-sqlite"
-version = "0.7.4"
-source = "git+https://github.com/kwannoel/sqlx.git?rev=ddf222f56cf99f865231a5383053645c6ea05ba3#ddf222f56cf99f865231a5383053645c6ea05ba3"
+version = "0.8.2"
+source = "git+https://github.com/madsim-rs/sqlx.git?rev=3efe6d0065963db2a2b7f30dee69f647e28dec81#3efe6d0065963db2a2b7f30dee69f647e28dec81"
 dependencies = [
- "async-channel 2.2.1",
  "atoi",
  "chrono",
  "flume",
@@ -13927,11 +13925,11 @@ dependencies = [
  "madsim-tokio",
  "percent-encoding",
  "serde",
+ "serde_urlencoded",
  "sqlx-core",
  "time",
  "tracing",
  "url",
- "urlencoding",
  "uuid",
 ]
 
@@ -14017,9 +14015,6 @@ name = "strum"
 version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125"
-dependencies = [
- "strum_macros 0.25.3",
-]
 
 [[package]]
 name = "strum"
@@ -14464,9 +14459,9 @@ dependencies = [
 
 [[package]]
 name = "tinyvec"
-version = "1.6.0"
+version = "1.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
 dependencies = [
  "tinyvec_macros",
 ]
@@ -14486,7 +14481,7 @@ dependencies = [
  "backtrace",
  "bytes",
  "libc",
- "mio",
+ "mio 0.8.11",
  "num_cpus",
  "parking_lot 0.12.1",
  "pin-project-lite",
@@ -14517,7 +14512,7 @@ dependencies = [
  "futures-util",
  "pin-project-lite",
  "tokio",
- "tokio-stream 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
+ "tokio-stream 0.1.16",
 ]
 
 [[package]]
@@ -14612,22 +14607,22 @@ dependencies = [
 [[package]]
 name = "tokio-stream"
 version = "0.1.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
+source = "git+https://github.com/madsim-rs/tokio.git?rev=0dd1055#0dd105567b323c863c29f794d2221ed588956d8d"
 dependencies = [
  "futures-core",
+ "madsim-tokio",
  "pin-project-lite",
- "tokio",
 ]
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.15"
-source = "git+https://github.com/madsim-rs/tokio.git?rev=0dd1055#0dd105567b323c863c29f794d2221ed588956d8d"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1"
 dependencies = [
  "futures-core",
- "madsim-tokio",
  "pin-project-lite",
+ "tokio",
 ]
 
 [[package]]
@@ -14717,9 +14712,9 @@ dependencies = [
 
 [[package]]
 name = "tonic"
-version = "0.12.2"
+version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6f6ba989e4b2c58ae83d862d3a3e27690b6e3ae630d0deb59f3697f32aa88ad"
+checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
 dependencies = [
  "async-stream",
  "async-trait",
@@ -14741,8 +14736,8 @@ dependencies = [
  "socket2 0.5.6",
  "tokio",
  "tokio-rustls 0.26.0",
- "tokio-stream 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)",
- "tower",
+ "tokio-stream 0.1.16",
+ "tower 0.4.13",
  "tower-layer",
  "tower-service",
  "tracing",
@@ -14782,11 +14777,25 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "tower"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36b837f86b25d7c0d7988f00a54e74739be6477f2aac6201b8f429a7569991b7"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project-lite",
+ "sync_wrapper",
+ "tower-layer",
+ "tower-service",
+]
+
 [[package]]
 name = "tower-http"
-version = "0.5.2"
+version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5"
+checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97"
 dependencies = [
  "async-compression",
  "bitflags 2.6.0",
@@ -14811,15 +14820,15 @@ dependencies = [
 
 [[package]]
 name = "tower-layer"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
 
 [[package]]
 name = "tower-service"
-version = "0.3.2"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
 
 [[package]]
 name = "tracing"
@@ -15342,11 +15351,11 @@ checksum = "5a84c137d37ab0142f0f2ddfe332651fdbf252e7b7dbb4e67b6c1f1b2e925101"
 
 [[package]]
 name = "value-trait"
-version = "0.8.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ea87257cfcbedcb9444eda79c59fdfea71217e6305afee8ee33f500375c2ac97"
+checksum = "9170e001f458781e92711d2ad666110f153e4e50bfd5cbd02db6547625714187"
 dependencies = [
- "float-cmp",
+ "float-cmp 0.10.0",
  "halfbrown",
  "itoa",
  "ryu",
@@ -16490,6 +16499,12 @@ version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
 
+[[package]]
+name = "yansi"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049"
+
 [[package]]
 name = "yup-oauth2"
 version = "8.3.0"
diff --git a/Cargo.toml b/Cargo.toml
index 78f02d46d60c2..0e1c1725baa8d 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -139,10 +139,10 @@ prost-build = { version = "0.13" }
 icelake = { git = "https://github.com/risingwavelabs/icelake.git", rev = "1783f8f106958d6d0ce0249c1c708934a15c2a47", features = [
     "prometheus",
 ] }
-# branch dev
-iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "f8b7bff7bd7853be7733fc1db1203862878f92c1" }
-iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "f8b7bff7bd7853be7733fc1db1203862878f92c1" }
-iceberg-catalog-glue = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "f8b7bff7bd7853be7733fc1db1203862878f92c1" }
+# branch dev-rebase-main-20241030
+iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "e28726443a57028f7c7df11d6d385470dc484d46" }
+iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "e28726443a57028f7c7df11d6d385470dc484d46" }
+iceberg-catalog-glue = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "e28726443a57028f7c7df11d6d385470dc484d46" }
 opendal = "0.49"
 # used only by arrow-udf-flight
 arrow-flight = "53"
@@ -153,15 +153,14 @@ arrow-udf-flight = "0.4"
 clap = { version = "4", features = ["cargo", "derive", "env"] }
 # Use a forked version which removes the dependencies on dynamo db to reduce
 # compile time and binary size.
-deltalake = { version = "0.20.1", features = [
-    "s3",
-    "gcs",
-    "datafusion",
-] }
+deltalake = { version = "0.20.1", features = ["s3", "gcs", "datafusion"] }
 itertools = "0.13.0"
 jsonbb = "0.1.4"
 lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "2682b85" }
-parquet = { version = "53", features = ["async"] }
+parquet = { version = "53.2", features = ["async"] }
+mysql_async = { version = "0.34", default-features = false, features = [
+    "default",
+] }
 thiserror-ext = "0.1.2"
 tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" }
 tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [
@@ -176,13 +175,13 @@ parking_lot = { version = "0.12", features = [
     "arc_lock",
     "deadlock_detection",
 ] }
-sea-orm = { version = "0.12.15", features = [
-    "sqlx-mysql",
-    "sqlx-postgres",
-    "sqlx-sqlite",
+sea-orm = { version = "~1.1", features = [
+    "sqlx-all",
     "runtime-tokio-native-tls",
+    "with-uuid",
 ] }
-sqlx = { version = "0.7.4", default-features = false, features = [
+sea-orm-migration = "~1.1"
+sqlx = { version = "0.8.2", default-features = false, features = [
     "bigdecimal",
     "chrono",
     "json",
@@ -285,6 +284,7 @@ new_without_default = "allow"
 # duplicated_attributes = "allow"
 # TODO: remove later https://github.com/rust-lang/rust-clippy/issues/12436
 mixed_attributes_style = "allow"
+too_long_first_doc_paragraph = "allow"
 
 [workspace.lints.rustdoc]
 private_intra_doc_links = "allow"
@@ -343,18 +343,16 @@ opt-level = 2
 
 [patch.crates-io]
 # Patch third-party crates for deterministic simulation.
-quanta = { git = "https://github.com/madsim-rs/quanta.git", rev = "948bdc3" }
+quanta = { git = "https://github.com/madsim-rs/quanta.git", rev = "ea9ba802327b1d72c4b1c7202c759b0a5243271e" }
 getrandom = { git = "https://github.com/madsim-rs/getrandom.git", rev = "e79a7ae" }
 # Don't patch `tokio-stream`, but only use the madsim version for **direct** dependencies.
 # Imagine an unpatched dependency depends on the original `tokio` and the patched `tokio-stream`.
 # tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "0dd1055" }
 tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = "95e2fd3" }
 tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88" }
-# NOTE(kwannoel): patch sqlx to use a version compatible with madsim.
-# If we can merge in: https://github.com/madsim-rs/sqlx/pull/2,
-# we can change it to patch madsim version instead.
-sqlx = { git = "https://github.com/kwannoel/sqlx.git", rev = "ddf222f56cf99f865231a5383053645c6ea05ba3" }
-madsim = { git = "https://github.com/madsim-rs/madsim.git", rev = "f0ffd24864245aea236833f1ff3638f75bd51700" }
+# sqlx version: v0.8.2
+# patch diffs: https://github.com/madsim-rs/sqlx/pull/4
+sqlx = { git = "https://github.com/madsim-rs/sqlx.git", rev = "3efe6d0065963db2a2b7f30dee69f647e28dec81" }
 futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" }
 # patch to remove preserve_order from serde_json
 bson = { git = "https://github.com/risingwavelabs/bson-rust", rev = "e5175ec" }
diff --git a/Makefile.toml b/Makefile.toml
index fb787603111ac..a0544239d4e12 100644
--- a/Makefile.toml
+++ b/Makefile.toml
@@ -27,7 +27,7 @@ env_scripts = [
 
   set_env ENABLE_TELEMETRY "false"
   set_env RW_TELEMETRY_TYPE "test"
-  set_env RW_SECRET_STORE_PRIVATE_KEY_HEX "0123456789abcdef"
+  set_env RW_SECRET_STORE_PRIVATE_KEY_HEX "0123456789abcdef0123456789abcdef"
   set_env RW_TEMP_SECRET_FILE_DIR "${PREFIX_SECRET}"
 
   is_sanitizer_enabled = get_env ENABLE_SANITIZER
diff --git a/ci/Dockerfile b/ci/Dockerfile
index 95dbb5205c754..88fe2519252e9 100644
--- a/ci/Dockerfile
+++ b/ci/Dockerfile
@@ -77,7 +77,7 @@ RUN cargo binstall -y --locked --no-symlinks cargo-llvm-cov cargo-nextest cargo-
     && rm -rf "/root/.cargo/registry/cache" \
     && rm -rf "/root/.cargo/git/db" \
     && cargo uninstall cargo-cache
-RUN cargo install cargo-dylint@3.1.0 dylint-link@3.1.0
+RUN cargo install cargo-dylint@3.2.1 dylint-link@3.2.1
 
 # install risedev
 COPY <<EOF /root/.cargo/bin/risedev
diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh
index 9d00b47bcd3aa..e08fc699df88f 100755
--- a/ci/build-ci-image.sh
+++ b/ci/build-ci-image.sh
@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240911
+export BUILD_ENV_VERSION=v20241030
 
 export BUILD_TAG="public.ecr.aws/w1p7b4n3/rw-build-env:${BUILD_ENV_VERSION}"
 
diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
index d89459bc02865..63bc3bee1f210 100644
--- a/ci/docker-compose.yml
+++ b/ci/docker-compose.yml
@@ -1,4 +1,5 @@
 services:
+  # TODO: Rename this to `postgres`
   db:
     image: postgres:15-alpine
     environment:
@@ -71,7 +72,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     depends_on:
       - mysql
       - sqlserver-server
@@ -85,7 +86,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     depends_on:
       - mysql
       - db
@@ -108,12 +109,20 @@ services:
 
 
   rw-build-env:
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
+    volumes:
+      - ..:/risingwave
+
+  pg-mysql-backend-test-env:
     image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    depends_on:
+      - mysql
+      - db
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -124,7 +133,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20240911
+    image: public.ecr.aws/w1p7b4n3/rw-build-env:v20241030
     depends_on:
       db:
         condition: service_healthy
diff --git a/ci/rust-toolchain b/ci/rust-toolchain
index 158ecbbdb0dfd..380c586961076 100644
--- a/ci/rust-toolchain
+++ b/ci/rust-toolchain
@@ -4,4 +4,4 @@
 # 3. (optional) **follow the instructions in lints/README.md** to update the toolchain and dependencies for lints
 
 [toolchain]
-channel = "nightly-2024-07-19"
+channel = "nightly-2024-10-11"
diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh
index c33387bfb9f3c..b1cda91b9cf7b 100755
--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@@ -27,17 +27,6 @@ if [[ "$profile" != "ci-dev" ]] && [[ "$profile" != "ci-release" ]]; then
     exit 1
 fi
 
-echo "--- Rust cargo-sort check"
-cargo sort --check --workspace --grouped
-
-# Disable hakari until we make sure it's useful
-# echo "--- Rust cargo-hakari check"
-# cargo hakari generate --diff
-# cargo hakari verify
-
-echo "--- Rust format check"
-cargo fmt --all -- --check
-
 echo "--- Build Rust components"
 
 if [[ "$profile" == "ci-dev" ]]; then
diff --git a/ci/scripts/check.sh b/ci/scripts/check.sh
index d51c2e13a6fb3..0457f24f45f65 100755
--- a/ci/scripts/check.sh
+++ b/ci/scripts/check.sh
@@ -23,6 +23,17 @@ configure_static_openssl
 echo "--- Run trailing spaces check"
 scripts/check/check-trailing-spaces.sh
 
+echo "--- Rust cargo-sort check"
+cargo sort --check --workspace --grouped
+
+# Disable hakari until we make sure it's useful
+# echo "--- Rust cargo-hakari check"
+# cargo hakari generate --diff
+# cargo hakari verify
+
+echo "--- Rust format check"
+cargo fmt --all -- --check
+
 echo "--- Run clippy check (dev, all features)"
 cargo clippy --all-targets --all-features --locked -- -D warnings
 
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
index 5a448a627926e..4ced7b94b7674 100755
--- a/ci/scripts/common.sh
+++ b/ci/scripts/common.sh
@@ -15,7 +15,7 @@ export MCLI_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/mc
 export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud-cli-475.0.0-linux-x86_64.tar.gz
 export NEXTEST_HIDE_PROGRESS_BAR=true
 export RW_TELEMETRY_TYPE=test
-export RW_SECRET_STORE_PRIVATE_KEY_HEX="0123456789abcdef"
+export RW_SECRET_STORE_PRIVATE_KEY_HEX="0123456789abcdef0123456789abcdef"
 export RUST_MIN_STACK=4194304
 
 unset LANG
diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh
index b0965688003c2..202e2dd1e2af1 100755
--- a/ci/scripts/deterministic-e2e-test.sh
+++ b/ci/scripts/deterministic-e2e-test.sh
@@ -30,25 +30,25 @@ export LOGDIR=.risingwave/log
 mkdir -p $LOGDIR
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, ddl"
-seq "$TEST_NUM" | parallel './risingwave_simulation ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/ddl-{}.log && rm $LOGDIR/ddl-{}.log'
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/ddl-{}.log && rm $LOGDIR/ddl-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, streaming"
-seq "$TEST_NUM" | parallel './risingwave_simulation ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/streaming-{}.log && rm $LOGDIR/streaming-{}.log'
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/streaming-{}.log && rm $LOGDIR/streaming-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, batch"
-seq "$TEST_NUM" | parallel './risingwave_simulation ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/batch-{}.log && rm $LOGDIR/batch-{}.log'
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/batch-{}.log && rm $LOGDIR/batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, kafka source"
-seq "$TEST_NUM" | parallel './risingwave_simulation --kafka-datadir=./e2e_test/source_legacy/basic/scripts/test_data ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log'
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation --kafka-datadir=./e2e_test/source_legacy/basic/scripts/test_data ./e2e_test/source_legacy/basic/kafka\*.slt 2> $LOGDIR/source-{}.log && rm $LOGDIR/source-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, streaming"
-seq "$TEST_NUM" | parallel './risingwave_simulation -j 16 ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/parallel-streaming-{}.log && rm $LOGDIR/parallel-streaming-{}.log'
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation -j 16 ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/parallel-streaming-{}.log && rm $LOGDIR/parallel-streaming-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch"
-seq "$TEST_NUM" | parallel './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (pre-generated-queries)"
-timeout 10m seq 64 | parallel RUST_MIN_STACK=4194304 './risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+timeout 10m seq 64 | parallel RUST_MIN_STACK=4194304 'MADSIM_TEST_SEED={} ./risingwave_simulation  --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, e2e extended mode test"
-seq "$TEST_NUM" | parallel 'RUST_LOG=info ./risingwave_simulation -e 2> $LOGDIR/extended-{}.log && rm $LOGDIR/extended-{}.log'
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} RUST_LOG=info ./risingwave_simulation -e 2> $LOGDIR/extended-{}.log && rm $LOGDIR/extended-{}.log'
diff --git a/ci/scripts/deterministic-it-test.sh b/ci/scripts/deterministic-it-test.sh
index 1190f8b0178f5..40288f5848b16 100755
--- a/ci/scripts/deterministic-it-test.sh
+++ b/ci/scripts/deterministic-it-test.sh
@@ -19,7 +19,7 @@ mv target/ci-sim target/sim
 TEST_PATTERN="$@"
 
 echo "--- Run integration tests in deterministic simulation mode"
-seq "$TEST_NUM" | parallel "NEXTEST_PROFILE=ci-sim \
+seq "$TEST_NUM" | parallel "MADSIM_TEST_SEED={} NEXTEST_PROFILE=ci-sim \
  cargo nextest run \
  --no-fail-fast \
  --cargo-metadata target/nextest/cargo-metadata.json \
diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh
index bf1f28861c24a..82641b66afc58 100755
--- a/ci/scripts/deterministic-recovery-test.sh
+++ b/ci/scripts/deterministic-recovery-test.sh
@@ -46,11 +46,7 @@ filter_stack_trace_for_all_logs() {
 
 # trap filter_stack_trace_for_all_logs ERR
 
-# NOTE(kwannoel): We must use `export` here, because the variables are not substituted
-# directly via bash subtitution. Instead, the `parallel` command substitutes the variables
-# from the environment. If they are declared without `export`, `parallel` can't read them from the env.
-export EXTRA_ARGS="--sqlite-data-dir=."
-
+export EXTRA_ARGS="${EXTRA_ARGS:-}"
 if [[ -n "${USE_ARRANGEMENT_BACKFILL:-}" ]]; then
   export EXTRA_ARGS="$EXTRA_ARGS --use-arrangement-backfill"
 fi
@@ -58,7 +54,7 @@ fi
 echo "--- EXTRA_ARGS: ${EXTRA_ARGS}"
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, background_ddl"
-seq "$TEST_NUM" | parallel './risingwave_simulation \
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 ${EXTRA_ARGS:-} \
@@ -66,7 +62,7 @@ ${EXTRA_ARGS:-} \
 2> $LOGDIR/recovery-background-ddl-{}.log && rm $LOGDIR/recovery-background-ddl-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, ddl"
-seq "$TEST_NUM" | parallel './risingwave_simulation \
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --background-ddl-rate=${BACKGROUND_DDL_RATE} \
@@ -74,7 +70,7 @@ ${EXTRA_ARGS:-} \
 ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, streaming"
-seq "$TEST_NUM" | parallel './risingwave_simulation \
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --background-ddl-rate=${BACKGROUND_DDL_RATE} \
@@ -82,7 +78,7 @@ ${EXTRA_ARGS:-} \
 ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, batch"
-seq "$TEST_NUM" | parallel './risingwave_simulation \
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --background-ddl-rate=${BACKGROUND_DDL_RATE} \
@@ -90,7 +86,7 @@ ${EXTRA_ARGS:-} \
 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-1meta, recovery, kafka source,sink"
-seq "$TEST_NUM" | parallel './risingwave_simulation \
+seq "$TEST_NUM" | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation \
 --kill \
 --kill-rate=${KILL_RATE} \
 --kafka-datadir=./e2e_test/source_legacy/basic/scripts/test_data \
diff --git a/ci/scripts/docker-scout-notify.sh b/ci/scripts/docker-scout-notify.sh
new file mode 100644
index 0000000000000..b81f21d6571f9
--- /dev/null
+++ b/ci/scripts/docker-scout-notify.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+buildkite-agent meta-data get SCOUT_REPORT > scout.report
+report=$(cat scout.report)
+cat >> step.yaml << EOF
+steps:
+  - label: "docker scout slack notification"
+    command: "echo '--- notify the scout report'"
+    notify:
+      - slack:
+          channels:
+            - "#notification-buildkite"
+          message: |
+            ${report}
+EOF
+
+buildkite-agent pipeline upload step.yaml
diff --git a/ci/scripts/docker-scout.sh b/ci/scripts/docker-scout.sh
new file mode 100644
index 0000000000000..1e64f58c3ffe9
--- /dev/null
+++ b/ci/scripts/docker-scout.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+ghcraddr="ghcr.io/risingwavelabs/risingwave"
+arch="$(uname -m)"
+image="${ghcraddr}:${BUILDKITE_COMMIT}-${arch}"
+
+echo "--- ghcr login"
+echo "$GHCR_TOKEN" | docker login ghcr.io -u "$GHCR_USERNAME" --password-stdin
+
+echo "--- dockerhub login"
+echo "$DOCKER_TOKEN" | docker login -u "risingwavelabs" --password-stdin
+
+echo "--- pull docker image"
+echo "pulling ${image}"
+docker pull "${image}"
+
+echo "--- check vulnerabilities"
+mkdir -p scout
+function docker-scout {
+    docker run -it -e DOCKER_SCOUT_HUB_USER=risingwavelabs -e DOCKER_SCOUT_HUB_PASSWORD=$DOCKER_TOKEN -u root -v /var/run/docker.sock:/var/run/docker.sock -v $PWD/scout:/scout docker/scout-cli "$@"
+}
+
+echo "--- scout quickview"
+docker-scout quickview ${image} -o /scout/quickview.output
+cat scout/quickview.output
+read C H M L <<< $(grep 'Target'  scout/quickview.output | awk -F'[ │ ]+' '{print $4, $5, $6, $7}' | sed 's/[CHML]//g')
+cat >> scout/report.output << EOF
+Docker Scout Report:
+            - Critical: $C
+            - High: $H
+            - Medium: $M
+            - Low: $L
+EOF
+
+buildkite-agent meta-data set "SCOUT_REPORT" "$(cat scout/report.output)"
+
+echo "--- scout recommendations"
+docker-scout recommendations "${image}"
+
+echo "--- scout cves"
+docker-scout cves "${image}"
diff --git a/ci/scripts/docker.sh b/ci/scripts/docker.sh
index e3080cff326a5..640d7af31dcba 100755
--- a/ci/scripts/docker.sh
+++ b/ci/scripts/docker.sh
@@ -29,6 +29,11 @@ docker buildx create \
   --name container \
   --driver=docker-container
 
+PULL_PARAM=""
+if [[ "${ALWAYS_PULL:-false}" = "true" ]]; then
+  PULL_PARAM="--pull"
+fi
+
 docker buildx build -f docker/Dockerfile \
   --build-arg "GIT_SHA=${BUILDKITE_COMMIT}" \
   --build-arg "CARGO_PROFILE=${CARGO_PROFILE}" \
@@ -36,6 +41,7 @@ docker buildx build -f docker/Dockerfile \
   --progress plain \
   --builder=container \
   --load \
+  ${PULL_PARAM} \
   --cache-to "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
   --cache-from "type=registry,ref=ghcr.io/risingwavelabs/risingwave-build-cache:${arch}" \
   .
diff --git a/ci/scripts/e2e-kafka-sink-test.sh b/ci/scripts/e2e-kafka-sink-test.sh
index e9dc660a95e67..d7b7821f1765d 100755
--- a/ci/scripts/e2e-kafka-sink-test.sh
+++ b/ci/scripts/e2e-kafka-sink-test.sh
@@ -150,7 +150,7 @@ echo "preparing confluent schema registry"
 python3 -m pip install --break-system-packages requests confluent-kafka
 
 echo "testing protobuf"
-sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/protobuf.slt'
+risedev slt 'e2e_test/sink/kafka/protobuf.slt'
 
 echo "testing avro"
-sqllogictest -p 4566 -d dev 'e2e_test/sink/kafka/avro.slt'
+risedev slt 'e2e_test/sink/kafka/avro.slt'
diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh
index 31ded0b59d4c8..4491db5633ea8 100755
--- a/ci/scripts/e2e-source-test.sh
+++ b/ci/scripts/e2e-source-test.sh
@@ -38,7 +38,8 @@ apt-get -y install jq
 echo "--- e2e, inline test"
 RUST_LOG="debug,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 risedev ci-start ci-inline-source-test
-risedev slt './e2e_test/source_inline/**/*.slt'
+risedev slt './e2e_test/source_inline/**/*.slt' -j16
+risedev slt './e2e_test/source_inline/**/*.slt.serial'
 echo "--- Kill cluster"
 risedev ci-kill
 
@@ -138,9 +139,6 @@ risedev slt './e2e_test/source_legacy/cdc/cdc.check_new_rows.slt'
 # drop relations
 risedev slt './e2e_test/source_legacy/cdc/cdc_share_stream_drop.slt'
 
-echo "--- postgres_query tvf test"
-risedev slt './e2e_test/source_legacy/tvf/postgres_query.slt'
-
 echo "--- Kill cluster"
 risedev ci-kill
 export RISINGWAVE_CI=true
diff --git a/ci/scripts/e2e-test-parallel.sh b/ci/scripts/e2e-test-parallel.sh
index c366ef07fc209..effe8e364dd24 100755
--- a/ci/scripts/e2e-test-parallel.sh
+++ b/ci/scripts/e2e-test-parallel.sh
@@ -45,7 +45,8 @@ kill_cluster
 echo "--- e2e, ci-3streaming-2serving-3fe, batch"
 RUST_LOG=$RUST_LOG \
 risedev ci-start ci-3streaming-2serving-3fe
-sqllogictest "${host_args[@]}" -d dev './e2e_test/ddl/**/*.slt' --junit "parallel-batch-ddl-${profile}" --label "parallel"
+# Exclude files that contain ALTER SYSTEM commands
+find ./e2e_test/ddl -name "*.slt" -type f -exec grep -L "ALTER SYSTEM" {} \; | xargs -r sqllogictest "${host_args[@]}" -d dev --junit "parallel-batch-ddl-${profile}" --label "parallel"
 sqllogictest "${host_args[@]}" -d dev './e2e_test/visibility_mode/*.slt' -j 16 --junit "parallel-batch-${profile}" --label "parallel"
 
 kill_cluster
diff --git a/ci/scripts/notify.py b/ci/scripts/notify.py
index 8823c2f0ee923..6427cff14cbdf 100755
--- a/ci/scripts/notify.py
+++ b/ci/scripts/notify.py
@@ -11,6 +11,7 @@
 MAIN_CRON_TEST_MAP = {
     "test-notify": ["noelkwan", "noelkwan"],
     "test-notify-2": ["noelkwan", "noelkwan"],
+    "test-notify-timeout": ["noelkwan", "noelkwan"],
     "docslt": ["tianxiao"],
     "e2e-test-release": ["zhi", "Eric"],
     "e2e-meta-backup-test-release": ["zhi", "Eric"],
@@ -30,8 +31,8 @@
     "recovery-test-deterministic": ["runji", "noelkwan"],
     "background-ddl-arrangement-backfill-recovery-test-deterministic": ["runji", "noelkwan"],
     "background-ddl-recovery-test-deterministic": ["runji", "noelkwan"],
-    "e2e-iceberg-sink-test": ["zilin"],
-    "e2e-iceberg-sink-v2-test": ["zilin"],
+    "e2e-iceberg-sink-test": ["zilin", "xinhao"],
+    "e2e-iceberg-sink-v2-test": ["zilin", "xinhao"],
     "e2e-java-binding-tests": ["yiming"],
     "s3-source-check-aws": ["bohan"],
     "s3-source-check-aws-json-parser": ["bohan"],
@@ -54,8 +55,9 @@
     "e2e-redis-sink-tests": ["xinhao"],
     "e2e-starrocks-sink-tests": ["xinhao"],
     "e2e-cassandra-sink-tests": ["xinhao"],
-    "e2e-clickhouse-sink-tests": ["bohan","xinhao"],
+    "e2e-clickhouse-sink-tests": ["bohan", "xinhao"],
     "e2e-pulsar-sink-tests": ["bohan"],
+    "e2e-mqtt-sink-tests": ["xinhao"],
     "connector-node-integration-test": ["siyuan"],
 }
 
@@ -110,7 +112,15 @@ def get_failed_tests(get_test_status, test_map):
     failed_test_map = {}
     for test in test_map.keys():
         test_status = get_test_status(test)
-        if test_status == "hard_failed" or test_status == "soft_failed":
+        if test_status == "hard_failed" or test_status == "soft_failed" or test_status == "errored":
+            print(f"{test} failed with outcome: {test_status}")
+            failed_test_map[test] = test_map[test]
+        elif test_status == "passed":
+            print(f"{test} passed with outcome: {test_status}")
+        elif test_status is None or test_status == "":
+            print(f"{test} no outcome, skipping")
+        else:
+            print(f"{test} failed with unknown outcome: {test_status}")
             failed_test_map[test] = test_map[test]
     return failed_test_map
 
@@ -188,14 +198,14 @@ def run_test_1():
 
 def main():
     test_map = get_test_map()
+    print("--- Getting failed tests")
     failed_test_map = get_failed_tests(get_buildkite_test_status, test_map)
     message = generate_test_status_message(failed_test_map)
     if message == "":
-        print("All tests passed, no need to notify")
+        print("--- Tests passed, no need to notify")
         return
     else:
-        print("Some tests failed, notify users")
-        print(message)
+        print("--- Some tests failed, notify users")
         cmd = format_cmd(message)
         print(cmd)
         subprocess.run(cmd, shell=True)
diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh
index b589b148dd945..46655c9633910 100755
--- a/ci/scripts/run-backfill-tests.sh
+++ b/ci/scripts/run-backfill-tests.sh
@@ -27,7 +27,7 @@ CLUSTER_PROFILE='ci-1cn-1fe-user-kafka-with-recovery'
 echo "--- Configuring cluster profiles"
 if [[ -n "${BUILDKITE:-}" ]]; then
   echo "Running in buildkite"
-  RUNTIME_CLUSTER_PROFILE='ci-3cn-1fe'
+  RUNTIME_CLUSTER_PROFILE='ci-backfill-3cn-1fe'
   MINIO_RATE_LIMIT_CLUSTER_PROFILE='ci-backfill-3cn-1fe-with-minio-rate-limit'
 else
   echo "Running locally"
diff --git a/ci/scripts/run-deterministic-fuzz-test.sh b/ci/scripts/run-deterministic-fuzz-test.sh
index 85965446b5fc6..720fef67b9453 100755
--- a/ci/scripts/run-deterministic-fuzz-test.sh
+++ b/ci/scripts/run-deterministic-fuzz-test.sh
@@ -30,4 +30,4 @@ download-and-decompress-artifact risingwave_simulation .
 chmod +x ./risingwave_simulation
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (seed)"
-seq 32 | parallel './risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
+seq 32 | parallel 'MADSIM_TEST_SEED={} ./risingwave_simulation --sqlsmith 100 ./src/tests/sqlsmith/tests/testdata 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log'
diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh
index c9a084396b296..b3956998bd0e9 100755
--- a/ci/scripts/run-e2e-test.sh
+++ b/ci/scripts/run-e2e-test.sh
@@ -35,8 +35,11 @@ cluster_start() {
     mkdir -p "$PREFIX_LOG"
     risedev clean-data
     risedev pre-start-dev
+    risedev dev standalone-minio-sqlite &
+    PID=$!
+    sleep 1
     start_standalone "$PREFIX_LOG"/standalone.log &
-    risedev dev standalone-minio-sqlite
+    wait $PID
   elif [[ $mode == "single-node" ]]; then
     mkdir -p "$PREFIX_LOG"
     risedev clean-data
@@ -45,6 +48,13 @@ cluster_start() {
     # Give it a while to make sure the single-node is ready.
     sleep 10
   else
+    # Initialize backends.
+    if [[ $mode == *"mysql-backend" ]]; then
+      mysql -h mysql -P 3306 -u root -p123456 -e "DROP DATABASE IF EXISTS metadata; CREATE DATABASE metadata;"
+    elif [[ $mode == *"pg-backend" ]]; then
+      PGPASSWORD=postgres psql -h db -p 5432 -U postgres -c "DROP DATABASE IF EXISTS metadata;" -c "CREATE DATABASE metadata;"
+    fi
+
     risedev ci-start "$mode"
   fi
 }
@@ -255,8 +265,11 @@ if [[ "$mode" == "standalone" ]]; then
   mkdir -p "$PREFIX_LOG"
   risedev clean-data
   risedev pre-start-dev
+  risedev dev standalone-minio-sqlite-compactor &
+  PID=$!
+  sleep 1
   start_standalone_without_compactor "$PREFIX_LOG"/standalone.log &
-  risedev dev standalone-minio-sqlite-compactor
+  wait $PID
   wait_standalone
   if compactor_is_online
   then
@@ -272,8 +285,11 @@ if [[ "$mode" == "standalone" ]]; then
   mkdir -p "$PREFIX_LOG"
   risedev clean-data
   risedev pre-start-dev
+  risedev dev standalone-minio-sqlite &
+  PID=$!
+  sleep 1
   start_standalone "$PREFIX_LOG"/standalone.log &
-  risedev dev standalone-minio-sqlite
+  wait $PID
   wait_standalone
   if ! compactor_is_online
   then
diff --git a/ci/workflows/docker.yml b/ci/workflows/docker.yml
index dce0b68f50850..603376ace665a 100644
--- a/ci/workflows/docker.yml
+++ b/ci/workflows/docker.yml
@@ -38,6 +38,7 @@ steps:
     depends_on:
       - "build-amd64"
       - "build-aarch64"
+    key: "multi-arch-image-create-push"
     plugins:
       - seek-oss/aws-sm#v2.3.1:
           env:
@@ -81,3 +82,23 @@ steps:
     agents:
       queue: "linux-arm64"
     retry: *auto-retry
+
+  - label: "docker scout"
+    if: build.env("ENABLE_DOCKER_SCOUT") == "true"
+    key: docker-scout
+    command: "ci/scripts/docker-scout.sh"
+    depends_on:
+      - "multi-arch-image-create-push"
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GHCR_USERNAME: ghcr-username
+            GHCR_TOKEN: ghcr-token
+            DOCKER_TOKEN: docker-token
+    retry: *auto-retry
+
+  - label: "generate notification step"
+    if: build.env("ENABLE_DOCKER_SCOUT") == "true"
+    depends_on:
+      - "docker-scout"
+    command: ci/scripts/docker-scout-notify.sh
diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
index e5fba261aa376..3f3cd705f09f9 100644
--- a/ci/workflows/main-cron.yml
+++ b/ci/workflows/main-cron.yml
@@ -466,29 +466,7 @@ steps:
 
   - label: "S3 source check on AWS (json parser)"
     key: "s3-v2-source-check-aws-json-parser"
-    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2.py -t json"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-s3-source-tests"
-      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
-    depends_on: build
-    plugins:
-      - seek-oss/aws-sm#v2.3.1:
-          env:
-            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-          environment:
-            - S3_SOURCE_TEST_CONF
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
-    retry: *auto-retry
-
-  - label: "S3 source new file check on AWS (json)"
-    key: "s3-v2-source-new-file-check-aws"
-    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2_new_file.py"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s file_source.py -t json"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-s3-source-tests"
@@ -510,51 +488,7 @@ steps:
 
   - label: "S3 sink on parquet and json file"
     key: "s3-sink-parquet-and-json-encode"
-    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_sink.py"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-s3-source-tests"
-      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
-    depends_on: build
-    plugins:
-      - seek-oss/aws-sm#v2.3.1:
-          env:
-            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-          environment:
-            - S3_SOURCE_TEST_CONF
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
-    retry: *auto-retry
-
-  - label: "S3 sink batching test"
-    key: "s3-sink-batching-test"
-    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_sink_batch.py"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-s3-source-tests"
-      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
-    depends_on: build
-    plugins:
-      - seek-oss/aws-sm#v2.3.1:
-          env:
-            S3_SOURCE_TEST_CONF: ci_s3_source_test_aws
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-          environment:
-            - S3_SOURCE_TEST_CONF
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
-    retry: *auto-retry
-
-  - label: "S3 source batch read on AWS (json parser)"
-    key: "s3-v2-source-batch-read-check-aws-json-parser"
-    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_batch.py -t json"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s file_sink.py"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-s3-source-tests"
@@ -576,7 +510,7 @@ steps:
 
   - label: "S3 source check on AWS (csv parser)"
     key: "s3-v2-source-check-aws-csv-parser"
-    command: "ci/scripts/s3-source-test.sh -p ci-release -s fs_source_v2.py -t csv_without_header"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s file_source.py -t csv_without_header"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-s3-source-tests"
@@ -596,23 +530,6 @@ steps:
     timeout_in_minutes: 25
     retry: *auto-retry
 
-  - label: "PosixFs source on OpenDAL fs engine (csv parser)"
-    key: "s3-source-test-for-opendal-fs-engine-csv-parser"
-    command: "ci/scripts/s3-source-test.sh -p ci-release -s posix_fs_source.py -t csv_without_header"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-s3-source-tests"
-      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
-    depends_on: build
-    plugins:
-      - docker-compose#v5.1.0:
-          run: rw-build-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
-    retry: *auto-retry
-
   - label: "pulsar source check"
     key: "pulsar-source-tests"
     command: "ci/scripts/pulsar-source-test.sh -p ci-release"
@@ -764,7 +681,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 25
+    timeout_in_minutes: 35
     retry: *auto-retry
 
   - label: "e2e single-node binary test"
@@ -787,6 +704,46 @@ steps:
     timeout_in_minutes: 25
     retry: *auto-retry
 
+  - label: "end-to-end test (postgres backend)"
+    key: e2e-test-postgres-backend
+    command: "ci/scripts/e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe-pg-backend"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-e2e-test-other-backends"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: pg-mysql-backend-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 25
+    retry: *auto-retry
+
+  - label: "end-to-end test (mysql backend)"
+    key: e2e-test-mysql-backend
+    command: "ci/scripts/e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe-mysql-backend"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/run-selected") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-e2e-test-other-backends"
+      || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: pg-mysql-backend-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 25
+    retry: *auto-retry
+
   - label: "end-to-end test for opendal (parallel)"
     key: "e2e-test-opendal-parallel"
     command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-release"
@@ -1179,6 +1136,13 @@ steps:
     command: |
       bash -c 'echo test && exit -1'
 
+  # Notification test.
+  - key: "test-notify-timeout"
+    if: build.pull_request.labels includes "ci/main-cron/test-notify" || build.env("CI_STEPS") =~ /(^|,)test_notify(,|$$)/
+    command: |
+      bash -c 'echo test && sleep 300'
+    timeout_in_minutes: 1
+
   - wait: true
     continue_on_failure: true
     allow_dependency_failure: true
@@ -1188,4 +1152,4 @@ steps:
   # This should be the LAST part of the main-cron file.
   - label: "trigger failed test notification"
     if: build.pull_request.labels includes "ci/main-cron/test-notify" || build.branch == "main"
-    command: "ci/scripts/notify.py"
\ No newline at end of file
+    command: "ci/scripts/notify.py"
diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml
index a2f926e9389d6..5924f0fe49bfa 100644
--- a/ci/workflows/pull-request.yml
+++ b/ci/workflows/pull-request.yml
@@ -201,7 +201,7 @@ steps:
           mount-buildkite-agent: true
           upload-container-logs: always
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 18
+    timeout_in_minutes: 20
     retry: *auto-retry
 
   - label: "end-to-end sink test"
@@ -771,7 +771,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 30
+    timeout_in_minutes: 40
     retry: *auto-retry
 
   - label: "e2e single-node binary test"
@@ -790,6 +790,38 @@ steps:
     timeout_in_minutes: 30
     retry: *auto-retry
 
+  - label: "end-to-end test (postgres backend)"
+    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe-pg-backend"
+    if: build.pull_request.labels includes "ci/run-e2e-test-other-backends" || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: pg-mysql-backend-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 30
+    retry: *auto-retry
+
+  - label: "end-to-end test (mysql backend)"
+    command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe-mysql-backend"
+    if: build.pull_request.labels includes "ci/run-e2e-test-other-backends" || build.env("CI_STEPS") =~ /(^|,)e2e-test-other-backends?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+      - "docslt"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: pg-mysql-backend-test-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 30
+    retry: *auto-retry
+
   # FIXME(kwannoel): Let the github PR labeller label it, if sqlsmith source files has changes.
   - label: "fuzz test"
     command: "ci/scripts/pr-fuzz-test.sh -p ci-dev"
diff --git a/dashboard/components/Relations.tsx b/dashboard/components/Relations.tsx
index 4636b3764947a..67feb2ac81cad 100644
--- a/dashboard/components/Relations.tsx
+++ b/dashboard/components/Relations.tsx
@@ -35,7 +35,7 @@ import Title from "../components/Title"
 import useFetch from "../lib/api/fetch"
 import {
   Relation,
-  StreamingJob,
+  StreamingRelation,
   getDatabases,
   getSchemas,
   getUsers,
@@ -73,7 +73,7 @@ export const dependentsColumn: Column<Relation> = {
   ),
 }
 
-export const fragmentsColumn: Column<StreamingJob> = {
+export const fragmentsColumn: Column<StreamingRelation> = {
   name: "Fragments",
   width: 1,
   content: (r) => (
@@ -101,6 +101,17 @@ export const primaryKeyColumn: Column<RwTable> = {
       .join(", "),
 }
 
+export const vnodeCountColumn: Column<RwTable> = {
+  name: "Vnode Count",
+  width: 1,
+  // The table catalogs retrieved here are constructed from SQL models,
+  // where the `vnode_count` column has already been populated during migration.
+  // Therefore, it should always be present and no need to specify a fallback.
+  content: (r) => r.maybeVnodeCount ?? "?",
+}
+
+export const tableColumns = [primaryKeyColumn, vnodeCountColumn]
+
 export const connectorColumnSource: Column<RwSource> = {
   name: "Connector",
   width: 3,
diff --git a/dashboard/lib/api/streaming.ts b/dashboard/lib/api/streaming.ts
index 211bd1b6bbc4c..ac7ba30d1258e 100644
--- a/dashboard/lib/api/streaming.ts
+++ b/dashboard/lib/api/streaming.ts
@@ -15,6 +15,7 @@
  *
  */
 
+import { Expose, plainToInstance } from "class-transformer"
 import _ from "lodash"
 import sortBy from "lodash/sortBy"
 import {
@@ -53,14 +54,6 @@ export async function getRelationIdInfos(): Promise<RelationIdInfos> {
   return fragmentIds
 }
 
-export async function getFragments(): Promise<TableFragments[]> {
-  let fragmentList: TableFragments[] = (await api.get("/fragments2")).map(
-    TableFragments.fromJSON
-  )
-  fragmentList = sortBy(fragmentList, (x) => x.tableId)
-  return fragmentList
-}
-
 export interface Relation {
   id: number
   name: string
@@ -75,7 +68,43 @@ export interface Relation {
   databaseName?: string
 }
 
-export interface StreamingJob extends Relation {
+export class StreamingJob {
+  @Expose({ name: "jobId" })
+  id!: number
+  @Expose({ name: "objType" })
+  _objType!: string
+  name!: string
+  jobStatus!: string
+  @Expose({ name: "parallelism" })
+  _parallelism!: any
+  maxParallelism!: number
+
+  get parallelism() {
+    const parallelism = this._parallelism
+    if (typeof parallelism === "string") {
+      // `Adaptive`
+      return parallelism
+    } else if (typeof parallelism === "object") {
+      // `Fixed (64)`
+      let key = Object.keys(parallelism)[0]
+      let value = parallelism[key]
+      return `${key} (${value})`
+    } else {
+      // fallback
+      return JSON.stringify(parallelism)
+    }
+  }
+
+  get type() {
+    if (this._objType == "Table") {
+      return "Table / MV"
+    } else {
+      return this._objType
+    }
+  }
+}
+
+export interface StreamingRelation extends Relation {
   dependentRelations: number[]
 }
 
@@ -98,17 +127,15 @@ export function relationTypeTitleCase(x: Relation) {
   return _.startCase(_.toLower(relationType(x)))
 }
 
-export function relationIsStreamingJob(x: Relation): x is StreamingJob {
+export function relationIsStreamingJob(x: Relation): x is StreamingRelation {
   const type = relationType(x)
   return type !== "UNKNOWN" && type !== "SOURCE" && type !== "INTERNAL"
 }
 
 export async function getStreamingJobs() {
-  let jobs = _.concat<StreamingJob>(
-    await getMaterializedViews(),
-    await getTables(),
-    await getIndexes(),
-    await getSinks()
+  let jobs = plainToInstance(
+    StreamingJob,
+    (await api.get("/streaming_jobs")) as any[]
   )
   jobs = sortBy(jobs, (x) => x.id)
   return jobs
diff --git a/dashboard/lib/layout.ts b/dashboard/lib/layout.ts
index e6549d4b1e979..714ed9fe0f0db 100644
--- a/dashboard/lib/layout.ts
+++ b/dashboard/lib/layout.ts
@@ -516,33 +516,3 @@ export function generateFragmentEdges(
   }
   return links
 }
-
-export function generateRelationBackPressureEdges(
-  layoutMap: RelationBoxPosition[]
-): Edge[] {
-  const links = []
-  const relationMap = new Map<string, RelationBoxPosition>()
-  for (const x of layoutMap) {
-    relationMap.set(x.id, x)
-  }
-  for (const relation of layoutMap) {
-    for (const parentId of relation.parentIds) {
-      const parentRelation = relationMap.get(parentId)!
-      links.push({
-        points: [
-          {
-            x: relation.x + relation.width / 2,
-            y: relation.y + relation.height / 2,
-          },
-          {
-            x: parentRelation.x + parentRelation.width / 2,
-            y: parentRelation.y + parentRelation.height / 2,
-          },
-        ],
-        source: relation.id,
-        target: parentId,
-      })
-    }
-  }
-  return links
-}
diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json
index 78bc775791ee2..ce861243e27cf 100644
--- a/dashboard/package-lock.json
+++ b/dashboard/package-lock.json
@@ -17,6 +17,7 @@
         "@uidotdev/usehooks": "^2.4.1",
         "base64url": "^3.0.1",
         "bootstrap-icons": "^1.9.1",
+        "class-transformer": "^0.5.1",
         "d3": "^7.6.1",
         "d3-axis": "^3.0.0",
         "d3-dag": "^0.11.4",
@@ -35,6 +36,7 @@
         "react-json-view": "^1.21.3",
         "react-syntax-highlighter": "^15.5.0",
         "recharts": "^2.3.2",
+        "reflect-metadata": "^0.2.2",
         "styled-components": "5.3.0",
         "ts-proto": "^1.169.1"
       },
@@ -4135,6 +4137,11 @@
       "integrity": "sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg==",
       "dev": true
     },
+    "node_modules/class-transformer": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz",
+      "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw=="
+    },
     "node_modules/classcat": {
       "version": "5.0.4",
       "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz",
@@ -9801,6 +9808,11 @@
       "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz",
       "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ=="
     },
+    "node_modules/reflect-metadata": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz",
+      "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q=="
+    },
     "node_modules/reflect.getprototypeof": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz",
@@ -14811,6 +14823,11 @@
       "integrity": "sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg==",
       "dev": true
     },
+    "class-transformer": {
+      "version": "0.5.1",
+      "resolved": "https://registry.npmjs.org/class-transformer/-/class-transformer-0.5.1.tgz",
+      "integrity": "sha512-SQa1Ws6hUbfC98vKGxZH3KFY0Y1lm5Zm0SY8XX9zbK7FJCyVEac3ATW0RIpwzW+oOfmHE5PMPufDG9hCfoEOMw=="
+    },
     "classcat": {
       "version": "5.0.4",
       "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz",
@@ -18962,6 +18979,11 @@
         }
       }
     },
+    "reflect-metadata": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.2.2.tgz",
+      "integrity": "sha512-urBwgfrvVP/eAyXx4hluJivBKzuEbSQs9rKWCrCkbSxNv8mxPcUZKeuoF3Uy4mJl3Lwprp6yy5/39VWigZ4K6Q=="
+    },
     "reflect.getprototypeof": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz",
diff --git a/dashboard/package.json b/dashboard/package.json
index 5e01f71c3686b..79897d63b19c8 100644
--- a/dashboard/package.json
+++ b/dashboard/package.json
@@ -24,6 +24,7 @@
     "@uidotdev/usehooks": "^2.4.1",
     "base64url": "^3.0.1",
     "bootstrap-icons": "^1.9.1",
+    "class-transformer": "^0.5.1",
     "d3": "^7.6.1",
     "d3-axis": "^3.0.0",
     "d3-dag": "^0.11.4",
@@ -42,6 +43,7 @@
     "react-json-view": "^1.21.3",
     "react-syntax-highlighter": "^15.5.0",
     "recharts": "^2.3.2",
+    "reflect-metadata": "^0.2.2",
     "styled-components": "5.3.0",
     "ts-proto": "^1.169.1"
   },
diff --git a/dashboard/pages/_app.tsx b/dashboard/pages/_app.tsx
index 25746318c1d24..8dd4d2cc1079d 100644
--- a/dashboard/pages/_app.tsx
+++ b/dashboard/pages/_app.tsx
@@ -15,6 +15,7 @@
  *
  */
 import "bootstrap-icons/font/bootstrap-icons.css"
+import "reflect-metadata"
 import "../styles/global.css"
 
 import { ChakraProvider } from "@chakra-ui/react"
diff --git a/dashboard/pages/fragment_graph.tsx b/dashboard/pages/fragment_graph.tsx
index 3ffbda24b232f..2312334e53abe 100644
--- a/dashboard/pages/fragment_graph.tsx
+++ b/dashboard/pages/fragment_graph.tsx
@@ -24,7 +24,12 @@ import {
   HStack,
   Input,
   Select,
+  Table,
+  TableContainer,
+  Tbody,
+  Td,
   Text,
+  Tr,
   VStack,
 } from "@chakra-ui/react"
 import * as d3 from "d3"
@@ -184,7 +189,7 @@ function buildFragmentDependencyAsEdges(
   return nodes
 }
 
-const SIDEBAR_WIDTH = 200
+const SIDEBAR_WIDTH = 225
 
 type BackPressureDataSource = "Embedded" | "Prometheus"
 const backPressureDataSources: BackPressureDataSource[] = [
@@ -202,23 +207,28 @@ interface EmbeddedBackPressureInfo {
 }
 
 export default function Streaming() {
-  const { response: relationList } = useFetch(getStreamingJobs)
+  const { response: streamingJobList } = useFetch(getStreamingJobs)
   const { response: relationIdInfos } = useFetch(getRelationIdInfos)
 
-  const [relationId, setRelationId] = useQueryState("id", parseAsInteger)
+  const [jobId, setJobId] = useQueryState("id", parseAsInteger)
   const [selectedFragmentId, setSelectedFragmentId] = useState<number>()
   const [tableFragments, setTableFragments] = useState<TableFragments>()
 
+  const job = useMemo(
+    () => streamingJobList?.find((j) => j.id === jobId),
+    [streamingJobList, jobId]
+  )
+
   const toast = useErrorToast()
 
   useEffect(() => {
-    if (relationId) {
+    if (jobId) {
       setTableFragments(undefined)
-      getFragmentsByJobId(relationId).then((tf) => {
+      getFragmentsByJobId(jobId).then((tf) => {
         setTableFragments(tf)
       })
     }
-  }, [relationId])
+  }, [jobId])
 
   const fragmentDependencyCallback = useCallback(() => {
     if (tableFragments) {
@@ -232,14 +242,14 @@ export default function Streaming() {
   }, [tableFragments])
 
   useEffect(() => {
-    if (relationList) {
-      if (!relationId) {
-        if (relationList.length > 0) {
-          setRelationId(relationList[0].id)
+    if (streamingJobList) {
+      if (!jobId) {
+        if (streamingJobList.length > 0) {
+          setJobId(streamingJobList[0].id)
         }
       }
     }
-  }, [relationId, relationList, setRelationId])
+  }, [jobId, streamingJobList, setJobId])
 
   // The table fragments of the selected fragment id
   const fragmentDependency = fragmentDependencyCallback()?.fragmentDep
@@ -276,7 +286,7 @@ export default function Streaming() {
         const fragmentIdToRelationId = map[relationId].map
         for (const fragmentId in fragmentIdToRelationId) {
           if (parseInt(fragmentId) == searchFragIdInt) {
-            setRelationId(parseInt(relationId))
+            setJobId(parseInt(relationId))
             setSelectedFragmentId(searchFragIdInt)
             return
           }
@@ -295,7 +305,7 @@ export default function Streaming() {
         for (const fragmentId in fragmentIdToRelationId) {
           let actorIds = fragmentIdToRelationId[fragmentId].ids
           if (actorIds.includes(searchActorIdInt)) {
-            setRelationId(parseInt(relationId))
+            setJobId(parseInt(relationId))
             setSelectedFragmentId(parseInt(fragmentId))
             return
           }
@@ -406,41 +416,70 @@ export default function Streaming() {
           height="full"
         >
           <FormControl>
-            <FormLabel>Relations</FormLabel>
+            <FormLabel>Streaming Jobs</FormLabel>
             <Input
               list="relationList"
               spellCheck={false}
               onChange={(event) => {
-                const id = relationList?.find(
+                const id = streamingJobList?.find(
                   (x) => x.name == event.target.value
                 )?.id
                 if (id) {
-                  setRelationId(id)
+                  setJobId(id)
                 }
               }}
               placeholder="Search..."
               mb={2}
             ></Input>
             <datalist id="relationList">
-              {relationList &&
-                relationList.map((r) => (
+              {streamingJobList &&
+                streamingJobList.map((r) => (
                   <option value={r.name} key={r.id}>
                     ({r.id}) {r.name}
                   </option>
                 ))}
             </datalist>
             <Select
-              value={relationId ?? undefined}
-              onChange={(event) => setRelationId(parseInt(event.target.value))}
+              value={jobId ?? undefined}
+              onChange={(event) => setJobId(parseInt(event.target.value))}
             >
-              {relationList &&
-                relationList.map((r) => (
+              {streamingJobList &&
+                streamingJobList.map((r) => (
                   <option value={r.id} key={r.name}>
                     ({r.id}) {r.name}
                   </option>
                 ))}
             </Select>
           </FormControl>
+          {job && (
+            <FormControl>
+              <FormLabel>Information</FormLabel>
+              <TableContainer>
+                <Table size="sm">
+                  <Tbody>
+                    <Tr>
+                      <Td fontWeight="medium">Type</Td>
+                      <Td isNumeric>{job.type}</Td>
+                    </Tr>
+                    <Tr>
+                      <Td fontWeight="medium">Status</Td>
+                      <Td isNumeric>{job.jobStatus}</Td>
+                    </Tr>
+                    <Tr>
+                      <Td fontWeight="medium">Parallelism</Td>
+                      <Td isNumeric>{job.parallelism}</Td>
+                    </Tr>
+                    <Tr>
+                      <Td fontWeight="medium" paddingEnd={0}>
+                        Max Parallelism
+                      </Td>
+                      <Td isNumeric>{job.maxParallelism}</Td>
+                    </Tr>
+                  </Tbody>
+                </Table>
+              </TableContainer>
+            </FormControl>
+          )}
           <FormControl>
             <FormLabel>Goto</FormLabel>
             <VStack spacing={2}>
diff --git a/dashboard/pages/indexes.tsx b/dashboard/pages/indexes.tsx
index 0e57e724dc206..0b0c6c9d1a195 100644
--- a/dashboard/pages/indexes.tsx
+++ b/dashboard/pages/indexes.tsx
@@ -16,15 +16,15 @@
  */
 
 import {
-  primaryKeyColumn,
   Relations,
   streamingJobColumns,
+  tableColumns,
 } from "../components/Relations"
 import { getIndexes } from "../lib/api/streaming"
 
 export default function Indexes() {
   return Relations("Indexes", getIndexes, [
     ...streamingJobColumns,
-    primaryKeyColumn,
+    ...tableColumns,
   ])
 }
diff --git a/dashboard/pages/internal_tables.tsx b/dashboard/pages/internal_tables.tsx
index 2bb0c66bf9c6c..f415641956e98 100644
--- a/dashboard/pages/internal_tables.tsx
+++ b/dashboard/pages/internal_tables.tsx
@@ -15,9 +15,9 @@
  *
  */
 
-import { primaryKeyColumn, Relations } from "../components/Relations"
+import { Relations, tableColumns } from "../components/Relations"
 import { getInternalTables } from "../lib/api/streaming"
 
 export default function InternalTables() {
-  return Relations("Internal Tables", getInternalTables, [primaryKeyColumn])
+  return Relations("Internal Tables", getInternalTables, [...tableColumns])
 }
diff --git a/dashboard/pages/materialized_views.tsx b/dashboard/pages/materialized_views.tsx
index d1bde8ac6c582..6c8adfcac8450 100644
--- a/dashboard/pages/materialized_views.tsx
+++ b/dashboard/pages/materialized_views.tsx
@@ -16,15 +16,15 @@
  */
 
 import {
-  primaryKeyColumn,
   Relations,
   streamingJobColumns,
+  tableColumns,
 } from "../components/Relations"
 import { getMaterializedViews } from "../lib/api/streaming"
 
 export default function MaterializedViews() {
   return Relations("Materialized Views", getMaterializedViews, [
     ...streamingJobColumns,
-    primaryKeyColumn,
+    ...tableColumns,
   ])
 }
diff --git a/dashboard/pages/tables.tsx b/dashboard/pages/tables.tsx
index c74d5b8ad4754..e580a493aefb9 100644
--- a/dashboard/pages/tables.tsx
+++ b/dashboard/pages/tables.tsx
@@ -17,9 +17,9 @@
 
 import {
   Column,
-  primaryKeyColumn,
   Relations,
   streamingJobColumns,
+  tableColumns,
 } from "../components/Relations"
 import { getTables } from "../lib/api/streaming"
 import { Table } from "../proto/gen/catalog"
@@ -34,6 +34,6 @@ export default function Tables() {
   return Relations("Tables", getTables, [
     associatedSourceColumn,
     ...streamingJobColumns,
-    primaryKeyColumn,
+    ...tableColumns,
   ])
 }
diff --git a/dashboard/tsconfig.json b/dashboard/tsconfig.json
index d88f23ca2cf51..42256872b36d1 100644
--- a/dashboard/tsconfig.json
+++ b/dashboard/tsconfig.json
@@ -18,6 +18,8 @@
     "isolatedModules": true,
     "jsx": "preserve",
     "incremental": true,
+    "experimentalDecorators": true,
+    "emitDecoratorMetadata": true,
   },
   "include": [
     "next-env.d.ts",
diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json
index 8efff55d606a3..2315a73da65e9 100644
--- a/docker/dashboards/risingwave-dev-dashboard.json
+++ b/docker/dashboards/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","query":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_inflight_barrier_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_in_flight_barrier {{table_id}}","metric":"","query":"meta_snapshot_backfill_inflight_barrier_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p50 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p90 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p99 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p999 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_pmax table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_snapshot_backfill_consume_snapshot_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} {{stage}} @ {{instance}}","metric":"","query":"rate(stream_snapshot_backfill_consume_snapshot_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p50 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p90 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p99 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p999 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_pmax table_id[{{table_id}}]","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p50 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p90 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p99 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p999 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_pmax table_id[{{table_id}}]","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_upstream_lag{job=~\"$job\",instance=~\"$node\"} / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag @ {{table_id}}","metric":"","query":"meta_snapshot_backfill_upstream_lag{job=~\"$job\",instance=~\"$node\"} / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Snapshot Backfill Lag","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":40,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed auto schema change attempts of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","query":"sum(auto_schema_change_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of succeeded auto schema change of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_success_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","query":"sum(auto_schema_change_success_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of Auto Schema Change Process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50{{table_id}} - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99{{table_id}} - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax{{table_id}} - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Latency (sec)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":48,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id)                             / ignoring (downstream_fragment_id) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)                             / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id)                             / ignoring (downstream_fragment_id) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)                             / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table total {{table_id}}: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{actor_id!='',job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} table {{table_id}}: {{desc}}","metric":"","query":"stream_memory_usage{actor_id!='',job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_compute_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_compute_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_same_result_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"same result count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_same_result_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Compute Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_state_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_state_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg state cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_state_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_state_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id, wait_side, executor)                             / ignoring (wait_side, executor) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment avg {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"sum(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id, wait_side, executor)                             / ignoring (wait_side, executor) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_align_duration_ns{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"rate(stream_barrier_align_duration_ns{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment total {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":75,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":92,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":99,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":106,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_hit{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_safe_version_hit{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_miss{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_safe_version_miss{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Safe Version Fetch Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":129,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":144,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing bloom_filter size, sstable file size, sstable block size etc.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Bloom Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable file size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable File Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable block size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Block Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing avg key and value count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Avg Key And Value Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":171,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":180,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","query":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","query":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_time_travel_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by time travel","metric":"","query":"storage_time_travel_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_split_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"split compaction group cg{{group}}","metric":"","query":"sum(storage_split_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_merge_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge compaction group cg{{group}}","metric":"","query":"sum(storage_merge_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Schedule","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The latency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_time_travel_version_replay_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_avg","metric":"","query":"rate(storage_time_travel_version_replay_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The frequency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_ops","metric":"","query":"sum(rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":229,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":239,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":243,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":247,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":255,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of valid and invalid subscription cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"invalid_subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"invalid_subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subsription Cursor Nums","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The subscription error num of cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subscription_cursor_error_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"subscription_cursor_error_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Error Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The amount of time a query exists inside the cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Query Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of declare","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Declare Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Since the last fetch, the time up to now","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Last Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":270,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":281,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rows sent by remote sink to the Java connector process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Remote Sink (Java) Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store unconsumed min epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} @ {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, sink_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, sink_id, actor_id, sink_name)","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":299,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":318,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":323,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":326,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}}","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":330,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":331,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":332,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"}) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write @ {{sink_name}}","metric":"","query":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"}) by (sink_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total write","metric":"","query":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":333,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_read_bytes{job=~\"$job\",instance=~\"$node\"}) by (table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read @ {{table_name}}","metric":"","query":"sum(iceberg_read_bytes{job=~\"$job\",instance=~\"$node\"}) by (table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(nimtable_read_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total read","metric":"","query":"sum(nimtable_read_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Read Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":334,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":335,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":336,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":337,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":338,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":339,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":340,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_inflight_barrier_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_in_flight_barrier {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p50 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p90 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p99 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p999 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_pmax table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Rows/sec that we read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_snapshot_backfill_consume_snapshot_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} {{stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Rows/sec that we read from the backfill snapshot by materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"\n                                sum by (table_id) (\n                                  rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\"}[$__rate_interval])\n                                )\n                                * on(table_id) group_left(table_name) (\n                                  group(table_info) by (table_name, table_id)\n                                )\n                            ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_name={{table_name}} table_id={{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows) by MV","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p50 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p90 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p99 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p999 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_pmax table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p50 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p90 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p99 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p999 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_pmax table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_upstream_lag{job=~\"$job\",instance=~\"$node\"} / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag @ {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Snapshot Backfill Lag","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":41,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed auto schema change attempts of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of succeeded auto schema change of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_success_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of Auto Schema Change Process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Latency (sec)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":49,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id)                             / ignoring (downstream_fragment_id) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)                             / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table total {{table_id}}: {{desc}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{actor_id!='',job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} table {{table_id}}: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_accessed_entry_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"accessed entry count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_compute_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_same_output_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"same output count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor State Computation","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_state_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_state_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg state cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id, wait_side, executor)                             / ignoring (wait_side, executor) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment avg {{fragment_id}} {{wait_side}} {{executor}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_align_duration_ns{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment total {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":76,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":93,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":100,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":107,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_hit{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_miss{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Safe Version Fetch Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":145,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing bloom_filter size, sstable file size, sstable block size etc.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Bloom Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable file size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable File Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable block size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Block Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing avg key and value count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Avg Key And Value Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compaction groups","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_compaction_group_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of compaction group","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compaction_group_size{job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group size - cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The file count of compaction group","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compaction_group_file_count{job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group file count - cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The throughput of compaction group","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":112},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compaction_group_throughput{job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group throughput - cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":120},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_split_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"split compaction group - left cg{{group}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_merge_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge compaction group - left cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Schedule","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":177,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":186,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":214,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_time_travel_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by time travel","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The latency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_time_travel_version_replay_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The frequency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_ops","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":234,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":241,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":244,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":248,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":252,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":260,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of valid and invalid subscription cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"invalid_subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subsription Cursor Nums","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The subscription error num of cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subscription_cursor_error_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Error Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The amount of time a query exists inside the cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Query Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of declare","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Declare Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Since the last fetch, the time up to now","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Last Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":275,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":286,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rows sent by remote sink to the Java connector process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Remote Sink (Java) Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store unconsumed min epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, sink_id, actor_id, sink_name)","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":304,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":318,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":323,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":326,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":330,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":331,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":332,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":333,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":334,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":335,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":336,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":337,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"}) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write @ {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total write","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":338,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_read_bytes{job=~\"$job\",instance=~\"$node\"}) by (table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read @ {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(nimtable_read_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total read","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Read Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":339,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":340,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":341,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":342,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":343,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":344,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":345,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/docker/dashboards/risingwave-user-dashboard.json b/docker/dashboards/risingwave-user-dashboard.json
index 2f19f6e5f6527..80b40dd535b9e 100644
--- a/docker/dashboards/risingwave-user-dashboard.json
+++ b/docker/dashboards/risingwave-user-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
diff --git a/docker/docker-compose-distributed-etcd.yml b/docker/docker-compose-distributed-etcd.yml
deleted file mode 100644
index 3f965c5a1b702..0000000000000
--- a/docker/docker-compose-distributed-etcd.yml
+++ /dev/null
@@ -1,380 +0,0 @@
----
-version: "3"
-x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
-services:
-  compactor-0:
-    <<: *image
-    command:
-      - compactor-node
-      - "--listen-addr"
-      - "0.0.0.0:6660"
-      - "--advertise-addr"
-      - "compactor-0:6660"
-      - "--prometheus-listener-addr"
-      - "0.0.0.0:1260"
-      - "--meta-address"
-      - "http://meta-node-0:5690"
-      - "--config-path"
-      - /risingwave.toml
-    expose:
-      - "6660"
-      - "1260"
-    ports: [ ]
-    depends_on:
-      - meta-node-0
-      # - minio-0
-    volumes:
-      - "./risingwave.toml:/risingwave.toml"
-    environment:
-      RUST_BACKTRACE: "1"
-      # If ENABLE_TELEMETRY is not set, telemetry will start by default
-      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
-    container_name: compactor-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/6660; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 1G
-  compute-node-0:
-    <<: *image
-    command:
-      - compute-node
-      - "--listen-addr"
-      - "0.0.0.0:5688"
-      - "--advertise-addr"
-      - "compute-node-0:5688"
-      - "--prometheus-listener-addr"
-      - "0.0.0.0:1222"
-      - "--meta-address"
-      - "http://meta-node-0:5690"
-      - "--config-path"
-      - /risingwave.toml
-    expose:
-      - "5688"
-      - "1222"
-    ports: [ ]
-    depends_on:
-      - meta-node-0
-      # - minio-0
-    volumes:
-      - "./risingwave.toml:/risingwave.toml"
-    environment:
-      RUST_BACKTRACE: "1"
-      # If ENABLE_TELEMETRY is not set, telemetry will start by default
-      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
-    container_name: compute-node-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5688; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-    deploy:
-      resources:
-        limits:
-          memory: 26G
-        reservations:
-          memory: 26G
-  etcd-0:
-    image: "quay.io/coreos/etcd:v3.5.10"
-    command:
-      - /usr/local/bin/etcd
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2388"
-      - "--advertise-client-urls"
-      - "http://etcd-0:2388"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2389"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd-0:2389"
-      - "--listen-metrics-urls"
-      - "http://0.0.0.0:2379"
-      - "--name"
-      - risedev-meta
-      - "--max-txn-ops"
-      - "999999"
-      - "--max-request-bytes"
-      - "10485760"
-      - "--auto-compaction-mode"
-      - periodic
-      - "--auto-compaction-retention"
-      - 1m
-      - "--snapshot-count"
-      - "10000"
-      - "--data-dir"
-      - /etcd-data
-    expose:
-      - "2388"
-    ports:
-      - "2388:2388"
-      - "2389:2389"
-    depends_on: [ ]
-    volumes:
-      - "etcd-0:/etcd-data"
-    environment: { }
-    container_name: etcd-0
-    healthcheck:
-      test:
-        - CMD
-        - etcdctl
-        - --endpoints=http://localhost:2388
-        - endpoint
-        - health
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-  frontend-node-0:
-    <<: *image
-    command:
-      - frontend-node
-      - "--listen-addr"
-      - "0.0.0.0:4566"
-      - "--meta-addr"
-      - "http://meta-node-0:5690"
-      - "--advertise-addr"
-      - "frontend-node-0:4566"
-      - "--config-path"
-      - /risingwave.toml
-      - "--prometheus-listener-addr"
-      - "0.0.0.0:2222"
-    expose:
-      - "4566"
-    ports:
-      - "4566:4566"
-    depends_on:
-      - meta-node-0
-    volumes:
-      - "./risingwave.toml:/risingwave.toml"
-    environment:
-      RUST_BACKTRACE: "1"
-      # If ENABLE_TELEMETRY is not set, telemetry will start by default
-      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
-    container_name: frontend-node-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 1G
-  grafana-0:
-    image: "grafana/grafana-oss:latest"
-    command: [ ]
-    expose:
-      - "3001"
-    ports:
-      - "3001:3001"
-    depends_on: [ ]
-    volumes:
-      - "grafana-0:/var/lib/grafana"
-      - "./grafana.ini:/etc/grafana/grafana.ini"
-      - "./grafana-risedev-datasource.yml:/etc/grafana/provisioning/datasources/grafana-risedev-datasource.yml"
-      - "./grafana-risedev-dashboard.yml:/etc/grafana/provisioning/dashboards/grafana-risedev-dashboard.yml"
-      - "./dashboards:/dashboards"
-    environment: { }
-    container_name: grafana-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/3001; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-  meta-node-0:
-    <<: *image
-    command:
-      - meta-node
-      - "--listen-addr"
-      - "0.0.0.0:5690"
-      - "--advertise-addr"
-      - "meta-node-0:5690"
-      - "--dashboard-host"
-      - "0.0.0.0:5691"
-      - "--prometheus-host"
-      - "0.0.0.0:1250"
-      - "--prometheus-endpoint"
-      - "http://prometheus-0:9500"
-      - "--backend"
-      - etcd
-      - "--etcd-endpoints"
-      - "etcd-0:2388"
-      - "--state-store"
-      - "hummock+minio://hummockadmin:hummockadmin@minio-0:9301/hummock001"
-      - "--data-directory"
-      - "hummock_001"
-      - "--config-path"
-      - /risingwave.toml
-    expose:
-      - "5690"
-      - "1250"
-      - "5691"
-    ports:
-      - "5690:5690"
-      - "5691:5691"
-    depends_on:
-      - "etcd-0"
-    volumes:
-      - "./risingwave.toml:/risingwave.toml"
-    environment:
-      RUST_BACKTRACE: "1"
-      # If ENABLE_TELEMETRY is not set, telemetry will start by default
-      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
-      RW_TELEMETRY_TYPE: ${RW_TELEMETRY_TYPE:-"docker-compose"}
-    container_name: meta-node-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5690; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-    deploy:
-      resources:
-        limits:
-          memory: 2G
-        reservations:
-          memory: 1G
-  minio-0:
-    image: "quay.io/minio/minio:latest"
-    command:
-      - server
-      - "--address"
-      - "0.0.0.0:9301"
-      - "--console-address"
-      - "0.0.0.0:9400"
-      - /data
-    expose:
-      - "9301"
-      - "9400"
-    ports:
-      - "9301:9301"
-      - "9400:9400"
-    depends_on: [ ]
-    volumes:
-      - "minio-0:/data"
-    entrypoint: "
-
-      /bin/sh -c '
-
-      set -e
-
-      mkdir -p \"/data/hummock001\"
-
-      /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\"
-
-      '"
-    environment:
-      MINIO_CI_CD: "1"
-      MINIO_PROMETHEUS_AUTH_TYPE: public
-      MINIO_PROMETHEUS_URL: "http://prometheus-0:9500"
-      MINIO_ROOT_PASSWORD: hummockadmin
-      MINIO_ROOT_USER: hummockadmin
-      MINIO_DOMAIN: "minio-0"
-    container_name: minio-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/9301; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-  prometheus-0:
-    image: "prom/prometheus:latest"
-    command:
-      - "--config.file=/etc/prometheus/prometheus.yml"
-      - "--storage.tsdb.path=/prometheus"
-      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
-      - "--web.console.templates=/usr/share/prometheus/consoles"
-      - "--web.listen-address=0.0.0.0:9500"
-      - "--storage.tsdb.retention.time=30d"
-    expose:
-      - "9500"
-    ports:
-      - "9500:9500"
-    depends_on: [ ]
-    volumes:
-      - "prometheus-0:/prometheus"
-      - "./prometheus.yaml:/etc/prometheus/prometheus.yml"
-    environment: { }
-    container_name: prometheus-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - sh -c 'printf "GET /-/healthy HTTP/1.0\n\n" | nc localhost 9500; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-  message_queue:
-    image: "redpandadata/redpanda:latest"
-    command:
-      - redpanda
-      - start
-      - "--smp"
-      - "1"
-      - "--reserve-memory"
-      - 0M
-      - "--memory"
-      - 4G
-      - "--overprovisioned"
-      - "--node-id"
-      - "0"
-      - "--check=false"
-      - "--kafka-addr"
-      - "PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092"
-      - "--advertise-kafka-addr"
-      - "PLAINTEXT://message_queue:29092,OUTSIDE://localhost:9092"
-    expose:
-      - "29092"
-      - "9092"
-      - "9644"
-    ports:
-      - "29092:29092"
-      - "9092:9092"
-      - "9644:9644"
-      - "8081:8081"
-    depends_on: [ ]
-    volumes:
-      - "message_queue:/var/lib/redpanda/data"
-    environment: { }
-    container_name: message_queue
-    healthcheck:
-      test: curl -f localhost:9644/v1/status/ready
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-volumes:
-  etcd-0:
-    external: false
-  grafana-0:
-    external: false
-  minio-0:
-    external: false
-  prometheus-0:
-    external: false
-  message_queue:
-    external: false
diff --git a/docker/docker-compose-distributed.yml b/docker/docker-compose-distributed.yml
index abe1f33ed9823..4be9fdbb57642 100644
--- a/docker/docker-compose-distributed.yml
+++ b/docker/docker-compose-distributed.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   compactor-0:
     <<: *image
@@ -19,7 +19,7 @@ services:
     expose:
       - "6660"
       - "1260"
-    ports: [ ]
+    ports: []
     depends_on:
       - meta-node-0
       # - minio-0
@@ -61,7 +61,7 @@ services:
     expose:
       - "5688"
       - "1222"
-    ports: [ ]
+    ports: []
     depends_on:
       - meta-node-0
       # - minio-0
@@ -148,12 +148,12 @@ services:
           memory: 1G
   grafana-0:
     image: "grafana/grafana-oss:latest"
-    command: [ ]
+    command: []
     expose:
       - "3001"
     ports:
       - "3001:3001"
-    depends_on: [ ]
+    depends_on: []
     volumes:
       - "grafana-0:/var/lib/grafana"
       - "./grafana.ini:/etc/grafana/grafana.ini"
@@ -241,7 +241,7 @@ services:
     ports:
       - "9301:9301"
       - "9400:9400"
-    depends_on: [ ]
+    depends_on: []
     volumes:
       - "minio-0:/data"
     entrypoint: "
@@ -284,7 +284,7 @@ services:
       - "9500"
     ports:
       - "9500:9500"
-    depends_on: [ ]
+    depends_on: []
     volumes:
       - "prometheus-0:/prometheus"
       - "./prometheus.yaml:/etc/prometheus/prometheus.yml"
@@ -326,7 +326,7 @@ services:
       - "9092:9092"
       - "9644:9644"
       - "8081:8081"
-    depends_on: [ ]
+    depends_on: []
     volumes:
       - "message_queue:/var/lib/redpanda/data"
     environment: {}
diff --git a/docker/docker-compose-etcd.yml b/docker/docker-compose-etcd.yml
deleted file mode 100644
index 9d6fe0566023e..0000000000000
--- a/docker/docker-compose-etcd.yml
+++ /dev/null
@@ -1,279 +0,0 @@
----
-version: "3"
-x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
-services:
-  risingwave-standalone:
-    <<: *image
-    command: "standalone --meta-opts=\" \
-                    --listen-addr 0.0.0.0:5690 \
-                    --advertise-addr 0.0.0.0:5690 \
-                    --dashboard-host 0.0.0.0:5691 \
-                    --prometheus-host 0.0.0.0:1250 \
-                    --prometheus-endpoint http://prometheus-0:9500 \
-                    --backend etcd \
-                    --etcd-endpoints etcd-0:2388 \
-                    --state-store hummock+minio://hummockadmin:hummockadmin@minio-0:9301/hummock001 \
-                    --data-directory hummock_001 \
-                    --config-path /risingwave.toml\" \
-                 --compute-opts=\" \
-                    --config-path /risingwave.toml \
-                    --listen-addr 0.0.0.0:5688 \
-                    --prometheus-listener-addr 0.0.0.0:1250 \
-                    --advertise-addr 0.0.0.0:5688 \
-                    --async-stack-trace verbose \
-                    #--parallelism 4 \
-                    #--total-memory-bytes 8589934592 \
-                    --role both \
-                    --meta-address http://0.0.0.0:5690\" \
-                 --frontend-opts=\" \
-                   --config-path /risingwave.toml \
-                   --listen-addr 0.0.0.0:4566 \
-                   --advertise-addr 0.0.0.0:4566 \
-                   --prometheus-listener-addr 0.0.0.0:1250 \
-                   --health-check-listener-addr 0.0.0.0:6786 \
-                   --meta-addr http://0.0.0.0:5690\" \
-                 --compactor-opts=\" \
-                   --listen-addr 0.0.0.0:6660 \
-                   --prometheus-listener-addr 0.0.0.0:1250 \
-                   --advertise-addr 0.0.0.0:6660 \
-                   --meta-address http://0.0.0.0:5690\""
-    expose:
-      - "6660"
-      - "4566"
-      - "5688"
-      - "5690"
-      - "1250"
-      - "5691"
-    ports:
-      - "4566:4566"
-      - "5690:5690"
-      - "5691:5691"
-      - "1250:1250"
-    depends_on:
-      - etcd-0
-      - minio-0
-    volumes:
-      - "./risingwave.toml:/risingwave.toml"
-    environment:
-      RUST_BACKTRACE: "1"
-      # If ENABLE_TELEMETRY is not set, telemetry will start by default
-      ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
-      RW_TELEMETRY_TYPE: ${RW_TELEMETRY_TYPE:-"docker-compose"}
-    container_name: risingwave-standalone
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/6660; exit $$?;'
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5688; exit $$?;'
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;'
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/5690; exit $$?;'
-      interval: 1s
-      timeout: 5s
-    restart: always
-    deploy:
-      resources:
-        limits:
-          memory: 28G
-        reservations:
-          memory: 28G
-
-  etcd-0:
-    image: "quay.io/coreos/etcd:v3.5.10"
-    command:
-      - /usr/local/bin/etcd
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2388"
-      - "--advertise-client-urls"
-      - "http://etcd-0:2388"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2389"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd-0:2389"
-      - "--listen-metrics-urls"
-      - "http://0.0.0.0:2379"
-      - "--name"
-      - risedev-meta
-      - "--max-txn-ops"
-      - "999999"
-      - "--max-request-bytes"
-      - "10485760"
-      - "--auto-compaction-mode"
-      - periodic
-      - "--auto-compaction-retention"
-      - 1m
-      - "--snapshot-count"
-      - "10000"
-      - "--data-dir"
-      - /etcd-data
-    expose:
-      - "2388"
-    ports:
-      - "2388:2388"
-      - "2389:2389"
-    depends_on: [ ]
-    volumes:
-      - "etcd-0:/etcd-data"
-    environment: { }
-    container_name: etcd-0
-    healthcheck:
-      test:
-        - CMD
-        - etcdctl
-        - --endpoints=http://localhost:2388
-        - endpoint
-        - health
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-
-  grafana-0:
-    image: "grafana/grafana-oss:latest"
-    command: [ ]
-    expose:
-      - "3001"
-    ports:
-      - "3001:3001"
-    depends_on: [ ]
-    volumes:
-      - "grafana-0:/var/lib/grafana"
-      - "./grafana.ini:/etc/grafana/grafana.ini"
-      - "./grafana-risedev-datasource.yml:/etc/grafana/provisioning/datasources/grafana-risedev-datasource.yml"
-      - "./grafana-risedev-dashboard.yml:/etc/grafana/provisioning/dashboards/grafana-risedev-dashboard.yml"
-      - "./dashboards:/dashboards"
-    environment: { }
-    container_name: grafana-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/3001; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-
-  minio-0:
-    image: "quay.io/minio/minio:latest"
-    command:
-      - server
-      - "--address"
-      - "0.0.0.0:9301"
-      - "--console-address"
-      - "0.0.0.0:9400"
-      - /data
-    expose:
-      - "9301"
-      - "9400"
-    ports:
-      - "9301:9301"
-      - "9400:9400"
-    depends_on: [ ]
-    volumes:
-      - "minio-0:/data"
-    entrypoint: "
-
-      /bin/sh -c '
-
-      set -e
-
-      mkdir -p \"/data/hummock001\"
-
-      /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\"
-
-      '"
-    environment:
-      MINIO_CI_CD: "1"
-      MINIO_PROMETHEUS_AUTH_TYPE: public
-      MINIO_PROMETHEUS_URL: "http://prometheus-0:9500"
-      MINIO_ROOT_PASSWORD: hummockadmin
-      MINIO_ROOT_USER: hummockadmin
-      MINIO_DOMAIN: "minio-0"
-    container_name: minio-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/9301; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-
-  prometheus-0:
-    image: "prom/prometheus:latest"
-    command:
-      - "--config.file=/etc/prometheus/prometheus.yml"
-      - "--storage.tsdb.path=/prometheus"
-      - "--web.console.libraries=/usr/share/prometheus/console_libraries"
-      - "--web.console.templates=/usr/share/prometheus/consoles"
-      - "--web.listen-address=0.0.0.0:9500"
-      - "--storage.tsdb.retention.time=30d"
-    expose:
-      - "9500"
-    ports:
-      - "9500:9500"
-    depends_on: [ ]
-    volumes:
-      - "prometheus-0:/prometheus"
-      - "./prometheus.yaml:/etc/prometheus/prometheus.yml"
-    environment: { }
-    container_name: prometheus-0
-    healthcheck:
-      test:
-        - CMD-SHELL
-        - sh -c 'printf "GET /-/healthy HTTP/1.0\n\n" | nc localhost 9500; exit $$?;'
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-
-  message_queue:
-    image: "redpandadata/redpanda:latest"
-    command:
-      - redpanda
-      - start
-      - "--smp"
-      - "1"
-      - "--reserve-memory"
-      - 0M
-      - "--memory"
-      - 4G
-      - "--overprovisioned"
-      - "--node-id"
-      - "0"
-      - "--check=false"
-      - "--kafka-addr"
-      - "PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092"
-      - "--advertise-kafka-addr"
-      - "PLAINTEXT://message_queue:29092,OUTSIDE://localhost:9092"
-    expose:
-      - "29092"
-      - "9092"
-      - "9644"
-    ports:
-      - "29092:29092"
-      - "9092:9092"
-      - "9644:9644"
-      - "8081:8081"
-    depends_on: [ ]
-    volumes:
-      - "message_queue:/var/lib/redpanda/data"
-    environment: { }
-    container_name: message_queue
-    healthcheck:
-      test: curl -f localhost:9644/v1/status/ready
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
-volumes:
-  etcd-0:
-    external: false
-  grafana-0:
-    external: false
-  minio-0:
-    external: false
-  prometheus-0:
-    external: false
-  message_queue:
-    external: false
diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml
index 1a0de9d31d169..99889d846dcb1 100644
--- a/docker/docker-compose-with-azblob.yml
+++ b/docker/docker-compose-with-azblob.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml
index 29a3d36b3eb65..80466c7cccaba 100644
--- a/docker/docker-compose-with-gcs.yml
+++ b/docker/docker-compose-with-gcs.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-hdfs.yml b/docker/docker-compose-with-hdfs.yml
index c8616e07a0853..7b69bdf3d29bc 100644
--- a/docker/docker-compose-with-hdfs.yml
+++ b/docker/docker-compose-with-hdfs.yml
@@ -17,7 +17,7 @@ services:
     expose:
       - "6660"
       - "1260"
-    ports: [ ]
+    ports: []
     depends_on:
       - meta-node-0
     volumes:
@@ -57,7 +57,7 @@ services:
     expose:
       - "5688"
       - "1222"
-    ports: [ ]
+    ports: []
     depends_on:
       - meta-node-0
     volumes:
@@ -80,56 +80,6 @@ services:
           memory: 26G
         reservations:
           memory: 26G
-  etcd-0:
-    image: "quay.io/coreos/etcd:v3.5.10"
-    command:
-      - /usr/local/bin/etcd
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2388"
-      - "--advertise-client-urls"
-      - "http://etcd-0:2388"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2389"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd-0:2389"
-      - "--listen-metrics-urls"
-      - "http://0.0.0.0:2379"
-      - "--name"
-      - risedev-meta
-      - "--max-txn-ops"
-      - "999999"
-      - "--max-request-bytes"
-      - "10485760"
-      - "--auto-compaction-mode"
-      - periodic
-      - "--auto-compaction-retention"
-      - 1m
-      - "--snapshot-count"
-      - "10000"
-      - "--data-dir"
-      - /etcd-data
-    expose:
-      - "2388"
-    ports:
-      - "2388:2388"
-      - "2389:2389"
-    depends_on: [ ]
-    volumes:
-      - "etcd-0:/etcd-data"
-    environment:
-      - HADOOP_HOME=/opt/hadoop/
-    container_name: etcd-0
-    healthcheck:
-      test:
-        - CMD
-        - etcdctl
-        - --endpoints=http://localhost:2388
-        - endpoint
-        - health
-      interval: 1s
-      timeout: 5s
-      retries: 5
-    restart: always
   frontend-node-0:
     image: "ghcr.io/risingwavelabs/risingwave:RisingWave_1.6.1_HDFS_2.7-x86_64"
     command:
@@ -171,19 +121,19 @@ services:
           memory: 1G
   grafana-0:
     image: "grafana/grafana-oss:latest"
-    command: [ ]
+    command: []
     expose:
       - "3001"
     ports:
       - "3001:3001"
-    depends_on: [ ]
+    depends_on: []
     volumes:
       - "grafana-0:/var/lib/grafana"
       - "./grafana.ini:/etc/grafana/grafana.ini"
       - "./grafana-risedev-datasource.yml:/etc/grafana/provisioning/datasources/grafana-risedev-datasource.yml"
       - "./grafana-risedev-dashboard.yml:/etc/grafana/provisioning/dashboards/grafana-risedev-dashboard.yml"
       - "./dashboards:/dashboards"
-    environment: { }
+    environment: {}
     container_name: grafana-0
     healthcheck:
       test:
@@ -208,9 +158,9 @@ services:
       - "--prometheus-endpoint"
       - "http://prometheus-0:9500"
       - "--backend"
-      - etcd
-      - "--etcd-endpoints"
-      - "etcd-0:2388"
+      - sql
+      - "--sql-endpoints"
+      - "postgres://postgres:@postgres-0:5432/metadata"
       - "--state-store"
       - "hummock+hdfs://<cluster_name>"
       - "--data-directory"
@@ -231,7 +181,7 @@ services:
       - "<HADOOP_HOME>:/opt/hadoop"
     environment:
       - HADOOP_HOME=/opt/hadoop/
-      - RW_TELEMETRY_TYPE: ${RW_TELEMETRY_TYPE:-"docker-compose"}
+      - RW_TELEMETRY_TYPE=${RW_TELEMETRY_TYPE:-"docker-compose"}
     container_name: meta-node-0
     healthcheck:
       test:
@@ -247,6 +197,19 @@ services:
           memory: 2G
         reservations:
           memory: 1G
+  postgres-0:
+    image: "postgres:15-alpine"
+    environment:
+      - POSTGRES_HOST_AUTH_METHOD=trust
+      - POSTGRES_USER=postgres
+      - POSTGRES_DB=metadata
+      - POSTGRES_INITDB_ARGS=--encoding=UTF-8 --lc-collate=C --lc-ctype=C
+    expose:
+      - "5432"
+    ports:
+      - "8432:5432"
+    volumes:
+      - "postgres-0:/var/lib/postgresql/data"
   prometheus-0:
     image: "prom/prometheus:latest"
     command:
@@ -260,11 +223,11 @@ services:
       - "9500"
     ports:
       - "9500:9500"
-    depends_on: [ ]
+    depends_on: []
     volumes:
       - "prometheus-0:/prometheus"
       - "./prometheus.yaml:/etc/prometheus/prometheus.yml"
-    environment: { }
+    environment: {}
     container_name: prometheus-0
     healthcheck:
       test:
@@ -302,10 +265,10 @@ services:
       - "9092:9092"
       - "9644:9644"
       - "8081:8081"
-    depends_on: [ ]
+    depends_on: []
     volumes:
       - "message_queue:/var/lib/redpanda/data"
-    environment: { }
+    environment: {}
     container_name: message_queue
     healthcheck:
       test: curl -f localhost:9644/v1/status/ready
@@ -314,7 +277,7 @@ services:
       retries: 5
     restart: always
 volumes:
-  etcd-0:
+  postgres-0:
     external: false
   grafana-0:
     external: false
diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml
index a4c0fc6d278c3..68483796ac800 100644
--- a/docker/docker-compose-with-local-fs.yml
+++ b/docker/docker-compose-with-local-fs.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml
index ea04d95e7e76a..f4bf8dc0e74c0 100644
--- a/docker/docker-compose-with-obs.yml
+++ b/docker/docker-compose-with-obs.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml
index 9e77aa6be8f9b..7d9563473182a 100644
--- a/docker/docker-compose-with-oss.yml
+++ b/docker/docker-compose-with-oss.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml
index 14f126bc39c10..d7dc75aa556a6 100644
--- a/docker/docker-compose-with-s3.yml
+++ b/docker/docker-compose-with-s3.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose-with-sqlite.yml b/docker/docker-compose-with-sqlite.yml
index d0bb26c42fe32..d4081b592c2ab 100644
--- a/docker/docker-compose-with-sqlite.yml
+++ b/docker/docker-compose-with-sqlite.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index ead28407a4016..e315878c98b77 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -1,6 +1,6 @@
 ---
 x-image: &image
-  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.0.1}
+  image: ${RW_IMAGE:-risingwavelabs/risingwave:v2.1.0-rc.1}
 services:
   risingwave-standalone:
     <<: *image
@@ -59,7 +59,7 @@ services:
       # If ENABLE_TELEMETRY is not set, telemetry will start by default
       ENABLE_TELEMETRY: ${ENABLE_TELEMETRY:-true}
       RW_TELEMETRY_TYPE: ${RW_TELEMETRY_TYPE:-"docker-compose"}
-      RW_SECRET_STORE_PRIVATE_KEY_HEX: ${RW_SECRET_STORE_PRIVATE_KEY_HEX:-0123456789abcdef}
+      RW_SECRET_STORE_PRIVATE_KEY_HEX: ${RW_SECRET_STORE_PRIVATE_KEY_HEX:-0123456789abcdef0123456789abcdef}
       RW_LICENSE_KEY: ${RW_LICENSE_KEY:-}
     container_name: risingwave-standalone
     healthcheck:
diff --git a/docker/prometheus.yaml b/docker/prometheus.yaml
index 23ca2884a96e4..7828eadca47f1 100644
--- a/docker/prometheus.yaml
+++ b/docker/prometheus.yaml
@@ -25,10 +25,6 @@ scrape_configs:
     static_configs:
       - targets: ["compactor-0:1260"]
 
-  - job_name: etcd
-    static_configs:
-      - targets: ["etcd-0:2379"]
-
   - job_name: frontend
     static_configs:
       - targets: ["frontend-node-0:2222"]
diff --git a/docs/dev/README.md b/docs/dev/README.md
index 7e47920d49400..ebfa30639f22d 100644
--- a/docs/dev/README.md
+++ b/docs/dev/README.md
@@ -12,7 +12,7 @@ Edit `SUMMARY.md` to add new chapters.
 
 ```sh
 # install tools
-> cargo binstall mdbook mdbook-linkcheck mdbook-toc
+> cargo install mdbook mdbook-linkcheck mdbook-toc
 # start a web server on localhost that you can visit to view the book,
 # and it will automatically reload each time you edit a page.
 > mdbook serve --open
diff --git a/e2e_test/background_ddl/basic.slt b/e2e_test/background_ddl/basic.slt
index 3c98b6943610e..c398a95cf1f5d 100644
--- a/e2e_test/background_ddl/basic.slt
+++ b/e2e_test/background_ddl/basic.slt
@@ -56,6 +56,48 @@ DROP MATERIALIZED VIEW m2;
 statement ok
 DROP TABLE t;
 
+################### Test stream now
+
+statement ok
+create table t (v1 timestamp with time zone, v2 timestamp with time zone);
+
+statement ok
+insert into t select to_timestamp(x) as z, to_timestamp(x) as y from generate_series(1, 1000) t(x);
+
+statement ok
+set backfill_rate_limit=1;
+
+statement ok
+create materialized view m1 as select * from t where v1 >= now() and v2 >= now();
+
+sleep 2s
+
+query I
+select count(*) from rw_catalog.rw_ddl_progress;
+----
+1
+
+statement ok
+recover;
+
+sleep 5s
+
+query I
+select count(*) from rw_catalog.rw_ddl_progress;
+----
+1
+
+statement ok
+drop materialized view m1;
+
+query I
+select count(*) from rw_catalog.rw_ddl_progress;
+----
+0
+
+statement ok
+drop table t;
+
 statement ok
 SET BACKGROUND_DDL=false;
 
diff --git a/e2e_test/batch/catalog/pg_settings.slt.part b/e2e_test/batch/catalog/pg_settings.slt.part
index 278436a884c70..73e84f371c35e 100644
--- a/e2e_test/batch/catalog/pg_settings.slt.part
+++ b/e2e_test/batch/catalog/pg_settings.slt.part
@@ -33,7 +33,6 @@ user create_compaction_group_for_mv
 user datestyle
 user enable_join_ordering
 user enable_share_plan
-user enable_shared_source
 user enable_two_phase_agg
 user extra_float_digits
 user force_split_distinct_agg
@@ -61,6 +60,7 @@ user streaming_max_parallelism
 user streaming_over_window_cache_policy
 user streaming_parallelism
 user streaming_use_arrangement_backfill
+user streaming_use_shared_source
 user streaming_use_snapshot_backfill
 user synchronize_seqscans
 user timezone
diff --git a/e2e_test/batch/explain.slt b/e2e_test/batch/explain.slt
index c3f4bfaf32f0e..1f9cebdf1b68b 100644
--- a/e2e_test/batch/explain.slt
+++ b/e2e_test/batch/explain.slt
@@ -11,4 +11,23 @@ statement ok
 drop table t;
 
 statement error Not supported: EXPLAIN CREATE VIEW
-explain create view v as select 1;
\ No newline at end of file
+explain create view v as select 1;
+
+query error
+explain (trace, format json) select 1;
+----
+    db error: ERROR: Failed to run the query
+
+Caused by:
+  Not supported: EXPLAIN (TRACE, JSON FORMAT)
+HINT: Only EXPLAIN (LOGICAL | PHYSICAL, JSON FORMAT) is supported.
+
+
+query error
+explain (distsql, format json) select 1;
+----
+    db error: ERROR: Failed to run the query
+
+Caused by:
+  Not supported: EXPLAIN (TRACE, JSON FORMAT)
+HINT: Only EXPLAIN (LOGICAL | PHYSICAL, JSON FORMAT) is supported.
\ No newline at end of file
diff --git a/e2e_test/commands/sr_register b/e2e_test/commands/sr_register
index 57dc65e50610d..7b29c317a26b8 100755
--- a/e2e_test/commands/sr_register
+++ b/e2e_test/commands/sr_register
@@ -4,24 +4,27 @@ set -euo pipefail
 
 # Register a schema to schema registry
 #
-# Usage: sr_register <subject> <schema>
+# Usage: sr_register <subject> <encoding>
+# Schema content is read from stdin. Use redirection or heredoc.
 #
 # https://docs.confluent.io/platform/current/schema-registry/develop/api.html#post--subjects-(string-%20subject)-versions
 
 # Validate arguments
 if [[ $# -ne 2 ]]; then
-    echo "Usage: sr_register <subject> <schema>"
+    echo "Usage: sr_register <subject> <encoding>"
+    echo "Schema content is read from stdin. Use redirection or heredoc."
+    echo "<encoding> is one of AVRO, PROTOBUF or JSON."
     exit 1
 fi
 
 subject="$1"
-schema="$2"
+encoding="$2"
 
 
-if [[ -z $subject || -z $schema ]]; then
+if [[ -z $subject || -z $encoding ]]; then
     echo "Error: Arguments cannot be empty"
     exit 1
 fi
 
-echo "$schema" | jq '{"schema": tojson}' \
-| curl -X POST -H 'content-type:application/vnd.schemaregistry.v1+json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/${subject}/versions"
+jq -Rs "{\"schema\": ., \"schemaType\": \"$encoding\"}" \
+| curl -X POST -H 'content-type:application/vnd.schemaregistry.v1+json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL:-http://schemaregistry:8082}/subjects/${subject}/versions"
diff --git a/e2e_test/ddl/alter_rename.slt b/e2e_test/ddl/alter_rename.slt
index b614d03e6929d..6d94bed9bb864 100644
--- a/e2e_test/ddl/alter_rename.slt
+++ b/e2e_test/ddl/alter_rename.slt
@@ -280,3 +280,43 @@ DROP TABLE t2;
 
 statement ok
 DROP TABLE t_as_1;
+
+# BEGIN references in changelog
+
+statement ok
+CREATE TABLE a (a0 int);
+
+statement ok
+CREATE TABLE b (b0 int);
+
+statement ok
+CREATE MATERIALIZED VIEW mv AS
+WITH a_log AS changelog FROM a,
+     b_log AS changelog FROM b
+SELECT 'a' AS tbl, * FROM a_log
+UNION ALL
+SELECT 'b', * FROM b_log;
+
+query T
+SELECT definition FROM rw_materialized_views WHERE name = 'mv';
+----
+CREATE MATERIALIZED VIEW mv AS WITH a_log AS changelog from a, b_log AS changelog from b SELECT 'a' AS tbl, * FROM a_log UNION ALL SELECT 'b', * FROM b_log
+
+statement ok
+ALTER TABLE a RENAME TO c;
+
+query T
+SELECT definition FROM rw_materialized_views WHERE name = 'mv';
+----
+CREATE MATERIALIZED VIEW mv AS WITH a_log AS changelog from "c", b_log AS changelog from b SELECT 'a' AS tbl, * FROM a_log UNION ALL SELECT 'b', * FROM b_log
+
+statement ok
+DROP MATERIALIZED VIEW mv;
+
+statement ok
+DROP TABLE b;
+
+statement ok
+DROP TABLE c;
+
+# END references in changelog
diff --git a/e2e_test/ddl/alter_set_schema.slt b/e2e_test/ddl/alter_set_schema.slt
index 74dcc5a77e64a..db0f479c85c05 100644
--- a/e2e_test/ddl/alter_set_schema.slt
+++ b/e2e_test/ddl/alter_set_schema.slt
@@ -94,23 +94,6 @@ WHERE nspname = 'test_schema';
 ----
 test_subscription test_schema
 
-statement ok
-CREATE CONNECTION test_conn WITH (type = 'privatelink', provider = 'mock');
-
-statement ok
-ALTER CONNECTION test_conn SET SCHEMA test_schema;
-
-query TT
-SELECT name AS connname, nspname AS schemaname
-FROM rw_connections
-JOIN pg_namespace ON pg_namespace.oid = rw_connections.schema_id
-WHERE nspname = 'test_schema';
-----
-test_conn test_schema
-
-statement ok
-DROP CONNECTION test_schema.test_conn;
-
 statement ok
 DROP SINK test_schema.test_sink;
 
diff --git a/e2e_test/ddl/alter_swap_rename.slt b/e2e_test/ddl/alter_swap_rename.slt
new file mode 100644
index 0000000000000..4accda4256951
--- /dev/null
+++ b/e2e_test/ddl/alter_swap_rename.slt
@@ -0,0 +1,181 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+# Create initial tables and views for testing swap
+statement ok
+CREATE TABLE t1 (v1 INT primary key, v2 STRUCT<v1 INT, v2 STRUCT<v1 INT, v2 INT>>);
+
+statement ok
+CREATE TABLE t2 (v1 INT primary key, v2 STRUCT<v1 INT, v2 STRUCT<v1 INT, v2 INT>>);
+
+# Insert some test data
+statement ok
+INSERT INTO t1 VALUES(1,(1,(1,2)));
+
+statement ok
+INSERT INTO t2 VALUES(2,(2,(2,4)));
+
+# Create materialized views referencing the tables
+statement ok
+CREATE MATERIALIZED VIEW mv1 AS SELECT v1, (t.v2).v1 AS v21 FROM t1 t;
+
+statement ok
+CREATE MATERIALIZED VIEW mv2 AS SELECT v1, (t.v2).v1 AS v21 FROM t2 t;
+
+# Create regular views
+statement ok
+CREATE VIEW v1 AS SELECT t1.v1 FROM t1;
+
+statement ok
+CREATE VIEW v2 AS SELECT t2.v2 FROM t2;
+
+# Create sources
+statement ok
+CREATE SOURCE src1 (v INT) WITH (
+    connector = 'datagen',
+    fields.v.kind = 'sequence',
+    fields.v.start = '1',
+    fields.v.end = '5',
+    datagen.rows.per.second='10',
+    datagen.split.num = '1'
+) FORMAT PLAIN ENCODE JSON;
+
+statement ok
+CREATE SOURCE src2 (v INT) WITH (
+    connector = 'datagen',
+    fields.v.kind = 'sequence',
+    fields.v.start = '6',
+    fields.v.end = '10',
+    datagen.rows.per.second='10',
+    datagen.split.num = '1'
+) FORMAT PLAIN ENCODE JSON;
+
+# Create sinks
+statement ok
+CREATE SINK sink1 AS SELECT * FROM mv1 WITH (
+    connector = 'blackhole'
+);
+
+statement ok
+CREATE SINK sink2 AS SELECT * FROM mv2 WITH (
+    connector = 'blackhole'
+);
+
+# Create subscriptions
+statement ok
+CREATE SUBSCRIPTION sub1 FROM mv1 WITH (
+    retention = '1D'
+);
+
+statement ok
+CREATE SUBSCRIPTION sub2 FROM mv2 WITH (
+    retention = '1D'
+);
+
+# Test table swap
+statement ok
+ALTER TABLE t1 SWAP WITH t2;
+
+statement error Permission denied
+ALTER TABLE t1 SWAP WITH mv1;
+
+statement error not found
+ALTER TABLE mv1 SWAP WITH mv2;
+
+query II rowsort
+SELECT * FROM t1;
+----
+2 (2,"(2,4)")
+
+query II rowsort
+SELECT * FROM t2;
+----
+1 (1,"(1,2)")
+
+# Test materialized view swap
+statement ok
+ALTER MATERIALIZED VIEW mv1 SWAP WITH mv2;
+
+# Verify materialized view contents
+query II rowsort
+SELECT * FROM mv1;
+----
+2 2
+
+query II rowsort
+SELECT * FROM mv2;
+----
+1 1
+
+# Test view swap
+statement ok
+ALTER VIEW v1 SWAP WITH v2;
+
+# Verify view definitions are swapped
+query TT
+SHOW CREATE VIEW v1;
+----
+public.v1 CREATE VIEW v1 AS SELECT t2.v2 FROM t1 AS t2
+
+query TT
+SHOW CREATE VIEW v2;
+----
+public.v2 CREATE VIEW v2 AS SELECT t1.v1 FROM t2 AS t1
+
+# Test source swap
+statement ok
+ALTER SOURCE src1 SWAP WITH src2;
+
+# Verify source definitions are swapped
+query TT
+SHOW CREATE SOURCE src1;
+----
+public.src1 CREATE SOURCE src1 (v INT) WITH (connector = 'datagen', fields.v.kind = 'sequence', fields.v.start = '6', fields.v.end = '10', datagen.rows.per.second = '10', datagen.split.num = '1') FORMAT PLAIN ENCODE JSON
+
+query TT
+SHOW CREATE SOURCE src2;
+----
+public.src2 CREATE SOURCE src2 (v INT) WITH (connector = 'datagen', fields.v.kind = 'sequence', fields.v.start = '1', fields.v.end = '5', datagen.rows.per.second = '10', datagen.split.num = '1') FORMAT PLAIN ENCODE JSON
+
+# Test sink swap
+statement ok
+ALTER SINK sink1 SWAP WITH sink2;
+
+# Verify sink definitions are swapped
+query TT
+SHOW CREATE SINK sink1;
+----
+public.sink1 CREATE SINK sink1 AS SELECT * FROM mv1 AS mv2 WITH (connector = 'blackhole')
+
+query TT
+SHOW CREATE SINK sink2;
+----
+public.sink2 CREATE SINK sink2 AS SELECT * FROM mv2 AS mv1 WITH (connector = 'blackhole')
+
+# Test subscription swap
+statement ok
+ALTER SUBSCRIPTION sub1 SWAP WITH sub2;
+
+# Verify subscription definitions are swapped
+query TT
+SHOW CREATE SUBSCRIPTION sub1;
+----
+public.sub1 CREATE SUBSCRIPTION sub1 FROM mv1 WITH (retention = '1D')
+
+query TT
+SHOW CREATE SUBSCRIPTION sub2;
+----
+public.sub2 CREATE SUBSCRIPTION sub2 FROM mv2 WITH (retention = '1D')
+
+# Clean up
+statement ok
+DROP SOURCE src1;
+
+statement ok
+DROP SOURCE src2;
+
+statement ok
+DROP TABLE t1 CASCADE;
+
+statement ok
+DROP TABLE t2 CASCADE;
diff --git a/e2e_test/ddl/connection.slt b/e2e_test/ddl/connection.slt
deleted file mode 100644
index 435395e9d249a..0000000000000
--- a/e2e_test/ddl/connection.slt
+++ /dev/null
@@ -1,23 +0,0 @@
-# Create a connection.
-statement ok
-CREATE CONNECTION conn0 WITH (type = 'privatelink', provider = 'mock');
-
-# Create another user with duplicate name.
-statement error
-CREATE CONNECTION conn0 WITH (type = 'privatelink', provider = 'mock');
-
-# Create if not exists.
-statement ok
-CREATE CONNECTION IF NOT EXISTS conn0 WITH (type = 'privatelink', provider = 'mock');
-
-# Test quoting.
-statement ok
-CREATE CONNECTION "conn1" WITH (type = 'privatelink', provider = 'mock');
-
-# Drop connections.
-statement ok
-DROP CONNECTION conn0;
-
-# Drop connections.
-statement ok
-DROP CONNECTION conn1;
diff --git a/e2e_test/ddl/database.slt b/e2e_test/ddl/database.slt
index 20080a84f1bf3..8f6c87a0d6509 100644
--- a/e2e_test/ddl/database.slt
+++ b/e2e_test/ddl/database.slt
@@ -10,6 +10,15 @@ create database ddl_database;
 statement ok
 create database if not exists ddl_database;
 
+statement ok
+create user user_for_db_owner;
+
+statement ok
+create database db_with_owner with owner = user_for_db_owner;
+
+statement ok
+create database db_with_owner2 owner user_for_db_owner;
+
 # Drop the database and public schema in it.
 statement ok
 drop database ddl_database;
@@ -21,3 +30,15 @@ drop database ddl_database;
 # Drop it again with if exists.
 statement ok
 drop database if exists ddl_database;
+
+# Drop the database with owner.
+statement ok
+drop database db_with_owner;
+
+# Drop the database with owner.
+statement ok
+drop database db_with_owner2;
+
+# Drop the user.
+statement ok
+drop user user_for_db_owner;
diff --git a/e2e_test/ddl/drop/drop_creating_mv.slt b/e2e_test/ddl/drop/drop_creating_mv.slt
index e9d8423f3cdec..678d880df90fd 100644
--- a/e2e_test/ddl/drop/drop_creating_mv.slt
+++ b/e2e_test/ddl/drop/drop_creating_mv.slt
@@ -18,11 +18,18 @@ risedev psql -c 'create materialized view m1 as select * from t;' &
 onlyif can-use-recover
 sleep 5s
 
+onlyif can-use-recover
+query I
+select background_ddl from rw_catalog.rw_materialized_views where name='m1';
+----
+f
+
 onlyif can-use-recover
 statement ok
 drop materialized view m1;
 
 ############## Test drop background mv BEFORE recovery
+
 statement ok
 set background_ddl=true;
 
@@ -33,6 +40,12 @@ create materialized view m1 as select * from t;
 onlyif can-use-recover
 sleep 5s
 
+onlyif can-use-recover
+query I
+select background_ddl from rw_catalog.rw_materialized_views where name='m1';
+----
+t
+
 onlyif can-use-recover
 statement ok
 drop materialized view m1;
@@ -48,12 +61,24 @@ create materialized view m1 as select * from t;
 onlyif can-use-recover
 sleep 5s
 
+onlyif can-use-recover
+query I
+select background_ddl from rw_catalog.rw_materialized_views where name='m1';
+----
+t
+
 onlyif can-use-recover
 statement ok
 recover;
 
 onlyif can-use-recover
-sleep 10s
+sleep 5s
+
+onlyif can-use-recover
+query I
+select background_ddl from rw_catalog.rw_materialized_views where name='m1';
+----
+t
 
 onlyif can-use-recover
 statement ok
@@ -69,6 +94,11 @@ set background_ddl=false;
 statement ok
 create materialized view m1 as select * from t;
 
+query I
+select background_ddl from rw_catalog.rw_materialized_views where name='m1';
+----
+f
+
 statement ok
 drop materialized view m1;
 
diff --git a/e2e_test/iceberg/start_spark_connect_server.sh b/e2e_test/iceberg/start_spark_connect_server.sh
index f0f3f19a1fab7..8f0c2640a1b59 100755
--- a/e2e_test/iceberg/start_spark_connect_server.sh
+++ b/e2e_test/iceberg/start_spark_connect_server.sh
@@ -3,7 +3,7 @@
 set -ex
 
 ICEBERG_VERSION=1.4.3
-SPARK_VERSION=3.4.3
+SPARK_VERSION=3.4.4
 
 PACKAGES="org.apache.iceberg:iceberg-spark-runtime-3.4_2.12:$ICEBERG_VERSION,org.apache.hadoop:hadoop-aws:3.3.2"
 PACKAGES="$PACKAGES,org.apache.spark:spark-connect_2.12:$SPARK_VERSION"
diff --git a/e2e_test/iceberg/test_case/iceberg_select_empty_table.slt b/e2e_test/iceberg/test_case/iceberg_select_empty_table.slt
index 832a7b781f7fb..ddfc07220f2b3 100644
--- a/e2e_test/iceberg/test_case/iceberg_select_empty_table.slt
+++ b/e2e_test/iceberg/test_case/iceberg_select_empty_table.slt
@@ -36,6 +36,7 @@ WITH (
     s3.region = 'us-east-1',
     s3.access.key = 'hummockadmin',
     s3.secret.key = 'hummockadmin',
+    s3.path.style.access = 'true',
     catalog.type = 'storage',
     warehouse.path = 's3a://icebergdata/demo',
     database.name = 'demo_db',
diff --git a/e2e_test/iceberg/test_case/iceberg_source_all_delete.slt b/e2e_test/iceberg/test_case/iceberg_source_all_delete.slt
index 6b4f984dabdc7..aa43c888b1507 100644
--- a/e2e_test/iceberg/test_case/iceberg_source_all_delete.slt
+++ b/e2e_test/iceberg/test_case/iceberg_source_all_delete.slt
@@ -22,25 +22,20 @@ CREATE SINK sink1 AS select * from mv1 WITH (
     s3.access.key = 'hummockadmin',
     s3.secret.key = 'hummockadmin',
     create_table_if_not_exists = 'true',
-    commit_checkpoint_interval = 5,
+    commit_checkpoint_interval = 3,
     primary_key = 'i1,i2',
 );
 
 statement ok
-INSERT INTO s1 (i1, i2, i3)
-SELECT s, s::text, s::text FROM generate_series(1, 10000) s;
+INSERT INTO s1 (i1, i2, i3) values(1,'1','1'),(2,'2','2'),(3,'3','3'),(4,'4','4'),(5,'5','5');
 
 statement ok
 flush
 
 statement ok
-DELETE FROM s1
-WHERE i1 IN (
-    SELECT s
-    FROM generate_series(1, 10000, 2) s
-);
+DELETE FROM s1 WHERE i1 < 3;
 
-sleep 10s
+sleep 16s
 
 statement ok
 CREATE SOURCE iceberg_t1_source
@@ -57,39 +52,23 @@ WITH (
 );
 
 statement ok
-DELETE FROM s1
-WHERE i1 IN (
-    SELECT s
-    FROM generate_series(1, 10000, 3) s
-);
+DELETE FROM s1 WHERE i1 > 4;
 
 statement ok
 flush
 
-sleep 15s
+sleep 16s
 
 query I
 select * from iceberg_t1_source order by i1 limit 5;
 ----
-2 2 2
-6 6 6
-8 8 8
-12 12 12
-14 14 14
-
-query I
-select * from iceberg_t1_source order by i1 desc limit 5;
-----
-9998 9998 9998
-9996 9996 9996
-9992 9992 9992
-9990 9990 9990
-9986 9986 9986
+3 3 3
+4 4 4
 
 query I
 select count(*) from iceberg_t1_source
 ----
-3333
+2
 
 statement ok
 DROP SINK sink1;
diff --git a/e2e_test/iceberg/test_case/iceberg_source_position_delete.slt b/e2e_test/iceberg/test_case/iceberg_source_position_delete.slt
index 40d665d7144e3..0109f39350752 100644
--- a/e2e_test/iceberg/test_case/iceberg_source_position_delete.slt
+++ b/e2e_test/iceberg/test_case/iceberg_source_position_delete.slt
@@ -22,25 +22,20 @@ CREATE SINK sink1 AS select * from mv1 WITH (
     s3.access.key = 'hummockadmin',
     s3.secret.key = 'hummockadmin',
     create_table_if_not_exists = 'true',
-    commit_checkpoint_interval = 5,
+    commit_checkpoint_interval = 3,
     primary_key = 'i1,i2',
 );
 
 statement ok
-INSERT INTO s1 (i1, i2, i3)
-SELECT s, s::text, s::text FROM generate_series(1, 10000) s;
+INSERT INTO s1 (i1, i2, i3) values(1,'1','1'),(2,'2','2'),(3,'3','3'),(4,'4','4'),(5,'5','5');
 
 statement ok
 flush
 
 statement ok
-DELETE FROM s1
-WHERE i1 IN (
-    SELECT s
-    FROM generate_series(1, 10000, 2) s
-);
+DELETE FROM s1 WHERE i1 < 3;
 
-sleep 15s
+sleep 16s
 
 statement ok
 CREATE SOURCE iceberg_t1_source
@@ -57,27 +52,16 @@ WITH (
 );
 
 query I
-select * from iceberg_t1_source order by i1 limit 5;
+select * from iceberg_t1_source order by i1;
 ----
-2 2 2
+3 3 3
 4 4 4
-6 6 6
-8 8 8
-10 10 10
-
-query I
-select * from iceberg_t1_source order by i1 desc limit 5;
-----
-10000 10000 10000
-9998 9998 9998
-9996 9996 9996
-9994 9994 9994
-9992 9992 9992
+5 5 5
 
 query I
 select count(*) from iceberg_t1_source
 ----
-5000
+3
 
 statement ok
 DROP SINK sink1;
diff --git a/e2e_test/over_window/generated/batch/main.slt.part b/e2e_test/over_window/generated/batch/main.slt.part
index 5e91838cfac89..dcd341e3e714f 100644
--- a/e2e_test/over_window/generated/batch/main.slt.part
+++ b/e2e_test/over_window/generated/batch/main.slt.part
@@ -9,3 +9,4 @@ include ./expr_in_win_func/mod.slt.part
 include ./agg_in_win_func/mod.slt.part
 include ./opt_agg_then_join/mod.slt.part
 include ./with_filter/mod.slt.part
+include ./no_effect_updates/mod.slt.part
diff --git a/e2e_test/over_window/generated/batch/no_effect_updates/mod.slt.part b/e2e_test/over_window/generated/batch/no_effect_updates/mod.slt.part
new file mode 100644
index 0000000000000..827b54113b962
--- /dev/null
+++ b/e2e_test/over_window/generated/batch/no_effect_updates/mod.slt.part
@@ -0,0 +1,94 @@
+# This file is generated by `gen.py`. Do not edit it manually!
+
+# Test handling of updates having no effect on window function outputs.
+
+statement ok
+create table t (
+      id int
+    , foo int
+    , bar int
+);
+
+statement ok
+create view v1 as
+select
+    *
+    , rank() over (partition by 1::int order by foo) as r1
+from t;
+
+statement ok
+create view v2 as
+select
+    *
+    , rank() over (partition by 1::int order by bar) as r2
+from t;
+
+statement ok
+insert into t values
+  (100001, 701, 805)
+, (100002, 700, 806)
+, (100003, 723, 807)
+, (100004, 702, 808);
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100001 701 805 2
+100004 702 808 3
+100003 723 807 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 701 805 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+update t set foo = 733 where id = 100001;
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100004 702 808 2
+100003 723 807 3
+100001 733 805 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 733 805 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+update t set bar = 804 where id = 100001;
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100004 702 808 2
+100003 723 807 3
+100001 733 804 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 733 804 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+drop view v1;
+
+statement ok
+drop view v2;
+
+statement ok
+drop table t;
diff --git a/e2e_test/over_window/generated/streaming/main.slt.part b/e2e_test/over_window/generated/streaming/main.slt.part
index 5e91838cfac89..dcd341e3e714f 100644
--- a/e2e_test/over_window/generated/streaming/main.slt.part
+++ b/e2e_test/over_window/generated/streaming/main.slt.part
@@ -9,3 +9,4 @@ include ./expr_in_win_func/mod.slt.part
 include ./agg_in_win_func/mod.slt.part
 include ./opt_agg_then_join/mod.slt.part
 include ./with_filter/mod.slt.part
+include ./no_effect_updates/mod.slt.part
diff --git a/e2e_test/over_window/generated/streaming/no_effect_updates/mod.slt.part b/e2e_test/over_window/generated/streaming/no_effect_updates/mod.slt.part
new file mode 100644
index 0000000000000..0261fbb1cc454
--- /dev/null
+++ b/e2e_test/over_window/generated/streaming/no_effect_updates/mod.slt.part
@@ -0,0 +1,94 @@
+# This file is generated by `gen.py`. Do not edit it manually!
+
+# Test handling of updates having no effect on window function outputs.
+
+statement ok
+create table t (
+      id int
+    , foo int
+    , bar int
+);
+
+statement ok
+create materialized view v1 as
+select
+    *
+    , rank() over (partition by 1::int order by foo) as r1
+from t;
+
+statement ok
+create materialized view v2 as
+select
+    *
+    , rank() over (partition by 1::int order by bar) as r2
+from t;
+
+statement ok
+insert into t values
+  (100001, 701, 805)
+, (100002, 700, 806)
+, (100003, 723, 807)
+, (100004, 702, 808);
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100001 701 805 2
+100004 702 808 3
+100003 723 807 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 701 805 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+update t set foo = 733 where id = 100001;
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100004 702 808 2
+100003 723 807 3
+100001 733 805 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 733 805 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+update t set bar = 804 where id = 100001;
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100004 702 808 2
+100003 723 807 3
+100001 733 804 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 733 804 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+drop materialized view v1;
+
+statement ok
+drop materialized view v2;
+
+statement ok
+drop table t;
diff --git a/e2e_test/over_window/templates/main.slt.part b/e2e_test/over_window/templates/main.slt.part
index 7f7ad74ab7424..e7ec434b8cfaf 100644
--- a/e2e_test/over_window/templates/main.slt.part
+++ b/e2e_test/over_window/templates/main.slt.part
@@ -7,3 +7,4 @@ include ./expr_in_win_func/mod.slt.part
 include ./agg_in_win_func/mod.slt.part
 include ./opt_agg_then_join/mod.slt.part
 include ./with_filter/mod.slt.part
+include ./no_effect_updates/mod.slt.part
diff --git a/e2e_test/over_window/templates/no_effect_updates/mod.slt.part b/e2e_test/over_window/templates/no_effect_updates/mod.slt.part
new file mode 100644
index 0000000000000..32627f5f38f0c
--- /dev/null
+++ b/e2e_test/over_window/templates/no_effect_updates/mod.slt.part
@@ -0,0 +1,92 @@
+# Test handling of updates having no effect on window function outputs.
+
+statement ok
+create table t (
+      id int
+    , foo int
+    , bar int
+);
+
+statement ok
+create $view_type v1 as
+select
+    *
+    , rank() over (partition by 1::int order by foo) as r1
+from t;
+
+statement ok
+create $view_type v2 as
+select
+    *
+    , rank() over (partition by 1::int order by bar) as r2
+from t;
+
+statement ok
+insert into t values
+  (100001, 701, 805)
+, (100002, 700, 806)
+, (100003, 723, 807)
+, (100004, 702, 808);
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100001 701 805 2
+100004 702 808 3
+100003 723 807 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 701 805 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+update t set foo = 733 where id = 100001;
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100004 702 808 2
+100003 723 807 3
+100001 733 805 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 733 805 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+update t set bar = 804 where id = 100001;
+
+query iii
+select * from v1 order by r1, id;
+----
+100002 700 806 1
+100004 702 808 2
+100003 723 807 3
+100001 733 804 4
+
+query iii
+select * from v2 order by r2, id;
+----
+100001 733 804 1
+100002 700 806 2
+100003 723 807 3
+100004 702 808 4
+
+statement ok
+drop $view_type v1;
+
+statement ok
+drop $view_type v2;
+
+statement ok
+drop table t;
diff --git a/e2e_test/s3/fs_sink.py b/e2e_test/s3/file_sink.py
similarity index 71%
rename from e2e_test/s3/fs_sink.py
rename to e2e_test/s3/file_sink.py
index 998c71e33eda6..a3113b430960a 100644
--- a/e2e_test/s3/fs_sink.py
+++ b/e2e_test/s3/file_sink.py
@@ -10,6 +10,8 @@
 from time import sleep
 from minio import Minio
 from random import uniform
+from time import sleep
+import time
 
 def gen_data(file_num, item_num_per_file):
     assert item_num_per_file % 2 == 0, \
@@ -265,8 +267,127 @@ def _assert_eq(field, got, expect):
     cur.execute(f'drop table test_parquet_sink_table')
     cur.execute(f'drop sink test_file_sink_json')
     cur.execute(f'drop table test_json_sink_table')
+    cur.execute(f'drop table s3_test_parquet')
+    cur.close()
+    conn.close()
+
+def test_file_sink_batching():
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+
+    # Open a cursor to execute SQL statements
+    cur = conn.cursor()
+
+
+    # Execute a SELECT statement
+    cur.execute(f'''CREATE TABLE t (v1 int, v2 int);''')
+
+    print('test file sink batching...\n')
+    cur.execute(f'''CREATE sink test_file_sink_batching as select
+        v1, v2 from t WITH (
+        connector = 's3',
+        s3.region_name = 'custom',
+        s3.bucket_name = 'hummock001',
+        s3.credentials.access = 'hummockadmin',
+        s3.credentials.secret = 'hummockadmin',
+        s3.endpoint_url = 'http://hummock001.127.0.0.1:9301',
+        s3.path = 'test_file_sink_batching/',
+        type = 'append-only',
+        rollover_seconds = 5,
+        max_row_count = 5,
+        force_append_only='true'
+    ) FORMAT PLAIN ENCODE PARQUET(force_append_only='true');''')
+
+    cur.execute(f'''CREATE TABLE test_file_sink_batching_table(
+        v1 int,
+        v2 int,
+    ) WITH (
+        connector = 's3',
+        match_pattern = 'test_file_sink_batching/*.parquet',
+        refresh.interval.sec = 1,
+        s3.region_name = 'custom',
+        s3.bucket_name = 'hummock001',
+        s3.credentials.access = 'hummockadmin',
+        s3.credentials.secret = 'hummockadmin',
+        s3.endpoint_url = 'http://hummock001.127.0.0.1:9301',
+    ) FORMAT PLAIN ENCODE PARQUET;''')
+
+    cur.execute(f'''ALTER SINK test_file_sink_batching SET PARALLELISM = 2;''')
+
+    cur.execute(f'''INSERT INTO t VALUES (10, 10);''')
+
+
+    cur.execute(f'select count(*) from test_file_sink_batching_table')
+    # no item will be selectedpsq
+    result = cur.fetchone()
+
+    def _assert_eq(field, got, expect):
+        assert got == expect, f'{field} assertion failed: got {got}, expect {expect}.'
+    def _assert_greater(field, got, expect):
+        assert got > expect, f'{field} assertion failed: got {got}, expect {expect}.'
+
+    _assert_eq('count(*)', result[0], 0)
+    print('the rollover_seconds has not reached, count(*) = 0')
+
+
+    time.sleep(11)
+
+    cur.execute(f'select count(*) from test_file_sink_batching_table')
+    result = cur.fetchone()
+    _assert_eq('count(*)', result[0], 1)
+    print('the rollover_seconds has reached, count(*) = ', result[0])
+
+    cur.execute(f'''
+    INSERT INTO t VALUES (20, 20);
+    INSERT INTO t VALUES (30, 30);
+    INSERT INTO t VALUES (40, 40);
+    INSERT INTO t VALUES (50, 10);
+    ''')
+
+    cur.execute(f'select count(*) from test_file_sink_batching_table')
+    # count(*) = 1
+    result = cur.fetchone()
+    _assert_eq('count(*)', result[0], 1)
+    print('the max row count has not reached, count(*) = ', result[0])
+
+    cur.execute(f'''
+    INSERT INTO t VALUES (60, 20);
+    INSERT INTO t VALUES (70, 30);
+    INSERT INTO t VALUES (80, 10);
+    INSERT INTO t VALUES (90, 20);
+    INSERT INTO t VALUES (100, 30);
+    INSERT INTO t VALUES (100, 10);
+    ''')
+
+    time.sleep(10)
+
+    cur.execute(f'select count(*) from test_file_sink_batching_table')
+    result = cur.fetchone()
+    _assert_greater('count(*)', result[0], 1)
+    print('the rollover_seconds has reached, count(*) = ', result[0])
+
+    cur.execute(f'drop sink test_file_sink_batching;')
+    cur.execute(f'drop table t;')
+    cur.execute(f'drop table test_file_sink_batching_table;')
     cur.close()
     conn.close()
+    # delete objects
+
+    client = Minio(
+        "127.0.0.1:9301",
+        "hummockadmin",
+        "hummockadmin",
+        secure=False,
+    )
+    objects = client.list_objects("hummock001", prefix="test_file_sink_batching/", recursive=True)
+
+    for obj in objects:
+        client.remove_object("hummock001", obj.object_name)
+        print(f"Deleted: {obj.object_name}")
 
 
 
@@ -306,7 +427,9 @@ def _assert_eq(field, got, expect):
 
     do_sink(config, FILE_NUM, ITEM_NUM_PER_FILE, run_id)
 
-     # clean up s3 files
+    # clean up s3 files
     for idx, _ in enumerate(data):
        client.remove_object("hummock001", _s3(idx))
 
+    # test file sink batching
+    test_file_sink_batching()
\ No newline at end of file
diff --git a/e2e_test/s3/fs_source_batch.py b/e2e_test/s3/file_source.py
similarity index 55%
rename from e2e_test/s3/fs_source_batch.py
rename to e2e_test/s3/file_source.py
index fc09b0ef4b516..0f3aaaed64a76 100644
--- a/e2e_test/s3/fs_source_batch.py
+++ b/e2e_test/s3/file_source.py
@@ -5,7 +5,9 @@
 import random
 import psycopg2
 
+# from handle_incremental_file import upload_to_s3_bucket, check_for_new_files
 from time import sleep
+import time
 from io import StringIO
 from minio import Minio
 from functools import partial
@@ -29,6 +31,7 @@ def format_json(data):
         for file in data
     ]
 
+
 def format_csv(data, with_header):
     csv_files = []
 
@@ -42,7 +45,7 @@ def format_csv(data, with_header):
         csv_files.append(ostream.getvalue())
     return csv_files
 
-def do_test(config, file_num, item_num_per_file, prefix, fmt):
+def do_test(config, file_num, item_num_per_file, prefix, fmt, need_drop_table=True):
     conn = psycopg2.connect(
         host="localhost",
         port="4566",
@@ -53,7 +56,7 @@ def do_test(config, file_num, item_num_per_file, prefix, fmt):
     # Open a cursor to execute SQL statements
     cur = conn.cursor()
 
-    def _source():
+    def _table():
         return f's3_test_{fmt}'
 
     def _encode():
@@ -62,33 +65,42 @@ def _encode():
         else:
             return f"CSV (delimiter = ',', without_header = {str('without' in fmt).lower()})"
 
+    def _include_clause():
+        if fmt == 'json':
+            return 'INCLUDE payload as rw_payload'
+        else:
+            return ''
+
     # Execute a SELECT statement
-    cur.execute(f'''CREATE SOURCE {_source()}(
+    cur.execute(f'''CREATE TABLE {_table()}(
         id int,
         name TEXT,
         sex int,
         mark int,
-    ) WITH (
+    )
+    {_include_clause()}
+    WITH (
         connector = 's3',
         match_pattern = '{prefix}*.{fmt}',
         s3.region_name = '{config['S3_REGION']}',
         s3.bucket_name = '{config['S3_BUCKET']}',
         s3.credentials.access = '{config['S3_ACCESS_KEY']}',
         s3.credentials.secret = '{config['S3_SECRET_KEY']}',
-        s3.endpoint_url = 'https://{config['S3_ENDPOINT']}'
+        s3.endpoint_url = 'https://{config['S3_ENDPOINT']}',
+        refresh.interval.sec = 1
     ) FORMAT PLAIN ENCODE {_encode()};''')
 
     total_rows = file_num * item_num_per_file
     MAX_RETRIES = 40
     for retry_no in range(MAX_RETRIES):
-        cur.execute(f'select count(*) from {_source()}')
+        cur.execute(f'select count(*) from {_table()}')
         result = cur.fetchone()
         if result[0] == total_rows:
             break
-        print(f"[retry {retry_no}] Now got {result[0]} rows in source, {total_rows} expected, wait 30s")
+        print(f"[retry {retry_no}] Now got {result[0]} rows in table, {total_rows} expected, wait 30s")
         sleep(30)
 
-    stmt = f'select count(*), sum(id), sum(sex), sum(mark) from {_source()}'
+    stmt = f'select count(*), sum(id), sum(sex), sum(mark) from {_table()}'
     print(f'Execute {stmt}')
     cur.execute(stmt)
     result = cur.fetchone()
@@ -103,13 +115,35 @@ def _assert_eq(field, got, expect):
     _assert_eq('sum(sex)', result[2], total_rows / 2)
     _assert_eq('sum(mark)', result[3], 0)
 
-    print('Test pass')
+    # only do payload check for json format, which enables INCLUDE CLAUSE
+    if fmt == 'json':
+        # check rw_payload
+        print('Check rw_payload')
+        stmt = f"select id, name, sex, mark, rw_payload from {_table()} limit 1;"
+        cur.execute(stmt)
+        result = cur.fetchone()
+        print("Got one line with rw_payload: ", result)
+        payload = result[4]
+        _assert_eq('id', payload['id'], result[0])
+        _assert_eq('name', payload['name'], result[1])
+        _assert_eq('sex', payload['sex'], result[2])
+        _assert_eq('mark', payload['mark'], result[3])
 
-    cur.execute(f'drop source {_source()}')
+        print('Test pass')
+
+    if need_drop_table:
+        cur.execute(f'drop table {_table()}')
     cur.close()
     conn.close()
 
-def test_empty_source(config, prefix, fmt):
+FORMATTER = {
+        'json': format_json,
+        'csv_with_header': partial(format_csv, with_header=True),
+        'csv_without_header': partial(format_csv, with_header=False),
+    }
+
+
+def test_batch_read(config, file_num, item_num_per_file, prefix, fmt):
     conn = psycopg2.connect(
         host="localhost",
         port="4566",
@@ -121,7 +155,7 @@ def test_empty_source(config, prefix, fmt):
     cur = conn.cursor()
 
     def _source():
-        return f's3_test_empty_{fmt}'
+        return f's3_test_{fmt}'
 
     def _encode():
         if fmt == 'json':
@@ -145,6 +179,16 @@ def _encode():
         s3.endpoint_url = 'https://{config['S3_ENDPOINT']}'
     ) FORMAT PLAIN ENCODE {_encode()};''')
 
+    total_rows = file_num * item_num_per_file
+    MAX_RETRIES = 40
+    for retry_no in range(MAX_RETRIES):
+        cur.execute(f'select count(*) from {_source()}')
+        result = cur.fetchone()
+        if result[0] == total_rows:
+            break
+        print(f"[retry {retry_no}] Now got {result[0]} rows in source, {total_rows} expected, wait 30s")
+        sleep(30)
+
     stmt = f'select count(*), sum(id), sum(sex), sum(mark) from {_source()}'
     print(f'Execute {stmt}')
     cur.execute(stmt)
@@ -155,25 +199,61 @@ def _encode():
     def _assert_eq(field, got, expect):
         assert got == expect, f'{field} assertion failed: got {got}, expect {expect}.'
 
-    _assert_eq('count(*)', result[0], 0)
+    _assert_eq('count(*)', result[0], total_rows)
+    _assert_eq('sum(id)', result[1], (total_rows - 1) * total_rows / 2)
+    _assert_eq('sum(sex)', result[2], total_rows / 2)
+    _assert_eq('sum(mark)', result[3], 0)
 
-    print('Empty source test pass')
+    print('Test batch read pass')
 
     cur.execute(f'drop source {_source()}')
     cur.close()
     conn.close()
 
+
+def upload_to_s3_bucket(config, minio_client, run_id, files, start_bias):
+    _local = lambda idx, start_bias: f"data_{idx + start_bias}.{fmt}"
+    _s3 = lambda idx, start_bias: f"{run_id}_data_{idx + start_bias}.{fmt}"
+    for idx, file_str in enumerate(files):
+        with open(_local(idx, start_bias), "w") as f:
+            f.write(file_str)
+            os.fsync(f.fileno())
+
+        minio_client.fput_object(
+            config["S3_BUCKET"], _s3(idx, start_bias), _local(idx, start_bias)
+        )
+
+
+def check_for_new_files(file_num, item_num_per_file, fmt):
+    conn = psycopg2.connect(host="localhost", port="4566", user="root", database="dev")
+
+    # Open a cursor to execute SQL statements
+    cur = conn.cursor()
+
+    def _table():
+        return f"s3_test_{fmt}"
+
+    total_rows = file_num * item_num_per_file
+
+    MAX_RETRIES = 40
+    for retry_no in range(MAX_RETRIES):
+        cur.execute(f"select count(*) from {_table()}")
+        result = cur.fetchone()
+        if result[0] == total_rows:
+            return True
+        print(
+            f"[retry {retry_no}] Now got {result[0]} rows in table, {total_rows} expected, wait 10s"
+        )
+        time.sleep(10)
+    return False
+
+
 if __name__ == "__main__":
     FILE_NUM = 4001
     ITEM_NUM_PER_FILE = 2
     data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
 
     fmt = sys.argv[1]
-    FORMATTER = {
-        'json': format_json,
-        'csv_with_header': partial(format_csv, with_header=True),
-        'csv_without_header': partial(format_csv, with_header=False),
-    }
     assert fmt in FORMATTER, f"Unsupported format: {fmt}"
     formatted_files = FORMATTER[fmt](data)
 
@@ -201,10 +281,41 @@ def _assert_eq(field, got, expect):
         )
 
     # do test
+    print("Test streaming file source...\n")
     do_test(config, FILE_NUM, ITEM_NUM_PER_FILE, run_id, fmt)
 
+    print("Test batch read file source...\n")
+    test_batch_read(config, FILE_NUM, ITEM_NUM_PER_FILE, run_id, fmt)
+
+    # test file source handle incremental files
+    data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
+    fmt = "json"
+
+    split_idx = 51
+    data_batch1 = data[:split_idx]
+    data_batch2 = data[split_idx:]
+    run_id = str(random.randint(1000, 9999))
+    print(f"S3 Source New File Test: run ID: {run_id} to buckek")
+
+    formatted_batch1 = FORMATTER[fmt](data_batch1)
+    upload_to_s3_bucket(config, client, run_id, formatted_batch1, 0)
+
+    # config in do_test that fs source's list interval is 1s
+    do_test(
+        config, len(data_batch1), ITEM_NUM_PER_FILE, run_id, fmt, need_drop_table=False
+    )
+
+    formatted_batch2 = FORMATTER[fmt](data_batch2)
+    upload_to_s3_bucket(config, client, run_id, formatted_batch2, split_idx)
+
+    success_flag = check_for_new_files(FILE_NUM, ITEM_NUM_PER_FILE, fmt)
+    if success_flag:
+        print("Test(add new file) pass")
+    else:
+        print("Test(add new file) fail")
+
+    _s3 = lambda idx, start_bias: f"{run_id}_data_{idx + start_bias}.{fmt}"
+
     # clean up s3 files
     for idx, _ in enumerate(formatted_files):
-        client.remove_object(config["S3_BUCKET"], _s3(idx))
-
-    test_empty_source(config, run_id, fmt)
\ No newline at end of file
+        client.remove_object(config["S3_BUCKET"], _s3(idx, 0))
diff --git a/e2e_test/s3/fs_sink_batch.py b/e2e_test/s3/fs_sink_batch.py
deleted file mode 100644
index 7bf41086294c8..0000000000000
--- a/e2e_test/s3/fs_sink_batch.py
+++ /dev/null
@@ -1,141 +0,0 @@
-import os
-import sys
-import random
-import psycopg2
-import json
-import time
-import pyarrow as pa
-import pyarrow.parquet as pq
-import pandas as pd
-from datetime import datetime, timezone
-from time import sleep
-from minio import Minio
-from random import uniform
-
-def do_test(config, file_num, item_num_per_file, prefix):
-    conn = psycopg2.connect(
-        host="localhost",
-        port="4566",
-        user="root",
-        database="dev"
-    )
-
-    # Open a cursor to execute SQL statements
-    cur = conn.cursor()
-
-
-    # Execute a SELECT statement
-    cur.execute(f'''CREATE TABLE t (v1 int, v2 int);''')
-
-    print('create sink')
-    cur.execute(f'''CREATE sink test_file_sink_batching as select
-        v1, v2 from t WITH (
-        connector = 's3',
-        s3.region_name = 'custom',
-        s3.bucket_name = 'hummock001',
-        s3.credentials.access = 'hummockadmin',
-        s3.credentials.secret = 'hummockadmin',
-        s3.endpoint_url = 'http://hummock001.127.0.0.1:9301',
-        s3.path = 'test_sink/',
-        type = 'append-only',
-        rollover_seconds = 5,
-        max_row_count = 5,
-        force_append_only='true'
-    ) FORMAT PLAIN ENCODE PARQUET(force_append_only='true');''')
-
-    cur.execute(f'''CREATE TABLE test_sink_table(
-        v1 int,
-        v2 int,
-    ) WITH (
-        connector = 's3',
-        match_pattern = 'test_sink/*.parquet',
-        refresh.interval.sec = 1,
-        s3.region_name = 'custom',
-        s3.bucket_name = 'hummock001',
-        s3.credentials.access = 'hummockadmin',
-        s3.credentials.secret = 'hummockadmin',
-        s3.endpoint_url = 'http://hummock001.127.0.0.1:9301',
-    ) FORMAT PLAIN ENCODE PARQUET;''')
-
-    cur.execute(f'''ALTER SINK test_file_sink_batching SET PARALLELISM = 2;''')
-
-    cur.execute(f'''INSERT INTO t VALUES (10, 10);''')
-
-
-    cur.execute(f'select count(*) from test_sink_table')
-    # no item will be selectedpsq
-    result = cur.fetchone()
-
-    def _assert_eq(field, got, expect):
-        assert got == expect, f'{field} assertion failed: got {got}, expect {expect}.'
-    def _assert_greater(field, got, expect):
-        assert got > expect, f'{field} assertion failed: got {got}, expect {expect}.'
-
-    _assert_eq('count(*)', result[0], 0)
-    print('the rollover_seconds has not reached, count(*) = 0')
-
-
-    time.sleep(11)
-
-    cur.execute(f'select count(*) from test_sink_table')
-    result = cur.fetchone()
-    _assert_eq('count(*)', result[0], 1)
-    print('the rollover_seconds has reached, count(*) = ', result[0])
-
-    cur.execute(f'''
-    INSERT INTO t VALUES (20, 20);
-    INSERT INTO t VALUES (30, 30);
-    INSERT INTO t VALUES (40, 40);
-    INSERT INTO t VALUES (50, 10);
-    ''')
-
-    cur.execute(f'select count(*) from test_sink_table')
-    # count(*) = 1
-    result = cur.fetchone()
-    _assert_eq('count(*)', result[0], 1)
-    print('the max row count has not reached, count(*) = ', result[0])
-
-    cur.execute(f'''
-    INSERT INTO t VALUES (60, 20);
-    INSERT INTO t VALUES (70, 30);
-    INSERT INTO t VALUES (80, 10);
-    INSERT INTO t VALUES (90, 20);
-    INSERT INTO t VALUES (100, 30);
-    INSERT INTO t VALUES (100, 10);
-    ''')
-
-    time.sleep(10)
-
-    cur.execute(f'select count(*) from test_sink_table')
-    result = cur.fetchone()
-    _assert_greater('count(*)', result[0], 1)
-    print('the rollover_seconds has reached, count(*) = ', result[0])
-
-    cur.execute(f'drop sink test_file_sink_batching;')
-    cur.execute(f'drop table t;')
-    cur.execute(f'drop table test_sink_table;')
-    cur.close()
-    conn.close()
-
-if __name__ == "__main__":
-    FILE_NUM = 10
-    ITEM_NUM_PER_FILE = 2000
-
-    config = json.loads(os.environ["S3_SOURCE_TEST_CONF"])
-    client = Minio(
-        "127.0.0.1:9301",
-        "hummockadmin",
-        "hummockadmin",
-        secure=False,
-    )
-    run_id = str(random.randint(1000, 9999))
-    _local = lambda idx: f'data_{idx}.parquet'
-    _s3 = lambda idx: f"{run_id}_data_{idx}.parquet"
-
-    do_test(config, FILE_NUM, ITEM_NUM_PER_FILE, run_id)
-
-    objects = client.list_objects("hummock001", prefix="test_sink/", recursive=True)
-
-    for obj in objects:
-        client.remove_object("hummock001", obj.object_name)
-        print(f"Deleted: {obj.object_name}")
diff --git a/e2e_test/s3/fs_source_v2.py b/e2e_test/s3/fs_source_v2.py
deleted file mode 100644
index 343b921540857..0000000000000
--- a/e2e_test/s3/fs_source_v2.py
+++ /dev/null
@@ -1,180 +0,0 @@
-import os
-import sys
-import csv
-import json
-import random
-import psycopg2
-
-from time import sleep
-from io import StringIO
-from minio import Minio
-from functools import partial
-
-def gen_data(file_num, item_num_per_file):
-    assert item_num_per_file % 2 == 0, \
-        f'item_num_per_file should be even to ensure sum(mark) == 0: {item_num_per_file}'
-    return [
-        [{
-            'id': file_id * item_num_per_file + item_id,
-            'name': f'{file_id}_{item_id}',
-            'sex': item_id % 2,
-            'mark': (-1) ** (item_id % 2),
-        } for item_id in range(item_num_per_file)]
-        for file_id in range(file_num)
-    ]
-
-def format_json(data):
-    return [
-        '\n'.join([json.dumps(item) for item in file])
-        for file in data
-    ]
-
-
-def format_csv(data, with_header):
-    csv_files = []
-
-    for file_data in data:
-        ostream = StringIO()
-        writer = csv.DictWriter(ostream, fieldnames=file_data[0].keys())
-        if with_header:
-            writer.writeheader()
-        for item_data in file_data:
-            writer.writerow(item_data)
-        csv_files.append(ostream.getvalue())
-    return csv_files
-
-def do_test(config, file_num, item_num_per_file, prefix, fmt, need_drop_table=True):
-    conn = psycopg2.connect(
-        host="localhost",
-        port="4566",
-        user="root",
-        database="dev"
-    )
-
-    # Open a cursor to execute SQL statements
-    cur = conn.cursor()
-
-    def _table():
-        return f's3_test_{fmt}'
-
-    def _encode():
-        if fmt == 'json':
-            return 'JSON'
-        else:
-            return f"CSV (delimiter = ',', without_header = {str('without' in fmt).lower()})"
-
-    def _include_clause():
-        if fmt == 'json':
-            return 'INCLUDE payload as rw_payload'
-        else:
-            return ''
-
-    # Execute a SELECT statement
-    cur.execute(f'''CREATE TABLE {_table()}(
-        id int,
-        name TEXT,
-        sex int,
-        mark int,
-    )
-    {_include_clause()}
-    WITH (
-        connector = 's3',
-        match_pattern = '{prefix}*.{fmt}',
-        s3.region_name = '{config['S3_REGION']}',
-        s3.bucket_name = '{config['S3_BUCKET']}',
-        s3.credentials.access = '{config['S3_ACCESS_KEY']}',
-        s3.credentials.secret = '{config['S3_SECRET_KEY']}',
-        s3.endpoint_url = 'https://{config['S3_ENDPOINT']}',
-        refresh.interval.sec = 1
-    ) FORMAT PLAIN ENCODE {_encode()};''')
-
-    total_rows = file_num * item_num_per_file
-    MAX_RETRIES = 40
-    for retry_no in range(MAX_RETRIES):
-        cur.execute(f'select count(*) from {_table()}')
-        result = cur.fetchone()
-        if result[0] == total_rows:
-            break
-        print(f"[retry {retry_no}] Now got {result[0]} rows in table, {total_rows} expected, wait 30s")
-        sleep(30)
-
-    stmt = f'select count(*), sum(id), sum(sex), sum(mark) from {_table()}'
-    print(f'Execute {stmt}')
-    cur.execute(stmt)
-    result = cur.fetchone()
-
-    print('Got:', result)
-
-    def _assert_eq(field, got, expect):
-        assert got == expect, f'{field} assertion failed: got {got}, expect {expect}.'
-
-    _assert_eq('count(*)', result[0], total_rows)
-    _assert_eq('sum(id)', result[1], (total_rows - 1) * total_rows / 2)
-    _assert_eq('sum(sex)', result[2], total_rows / 2)
-    _assert_eq('sum(mark)', result[3], 0)
-
-    # only do payload check for json format, which enables INCLUDE CLAUSE
-    if fmt == 'json':
-        # check rw_payload
-        print('Check rw_payload')
-        stmt = f"select id, name, sex, mark, rw_payload from {_table()} limit 1;"
-        cur.execute(stmt)
-        result = cur.fetchone()
-        print("Got one line with rw_payload: ", result)
-        payload = result[4]
-        _assert_eq('id', payload['id'], result[0])
-        _assert_eq('name', payload['name'], result[1])
-        _assert_eq('sex', payload['sex'], result[2])
-        _assert_eq('mark', payload['mark'], result[3])
-
-        print('Test pass')
-
-    if need_drop_table:
-        cur.execute(f'drop table {_table()}')
-    cur.close()
-    conn.close()
-
-FORMATTER = {
-        'json': format_json,
-        'csv_with_header': partial(format_csv, with_header=True),
-        'csv_without_header': partial(format_csv, with_header=False),
-    }
-
-if __name__ == "__main__":
-    FILE_NUM = 4001
-    ITEM_NUM_PER_FILE = 2
-    data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
-
-    fmt = sys.argv[1]
-    assert fmt in FORMATTER, f"Unsupported format: {fmt}"
-    formatted_files = FORMATTER[fmt](data)
-
-    config = json.loads(os.environ["S3_SOURCE_TEST_CONF"])
-    client = Minio(
-        config["S3_ENDPOINT"],
-        access_key=config["S3_ACCESS_KEY"],
-        secret_key=config["S3_SECRET_KEY"],
-        secure=True,
-    )
-    run_id = str(random.randint(1000, 9999))
-    _local = lambda idx: f'data_{idx}.{fmt}'
-    _s3 = lambda idx: f"{run_id}_data_{idx}.{fmt}"
-
-    # put s3 files
-    for idx, file_str in enumerate(formatted_files):
-        with open(_local(idx), "w") as f:
-            f.write(file_str)
-            os.fsync(f.fileno())
-
-        client.fput_object(
-            config["S3_BUCKET"],
-            _s3(idx),
-            _local(idx)
-        )
-
-    # do test
-    do_test(config, FILE_NUM, ITEM_NUM_PER_FILE, run_id, fmt)
-
-    # clean up s3 files
-    for idx, _ in enumerate(formatted_files):
-        client.remove_object(config["S3_BUCKET"], _s3(idx))
diff --git a/e2e_test/s3/fs_source_v2_new_file.py b/e2e_test/s3/fs_source_v2_new_file.py
deleted file mode 100644
index c90103e15c127..0000000000000
--- a/e2e_test/s3/fs_source_v2_new_file.py
+++ /dev/null
@@ -1,90 +0,0 @@
-from fs_source_v2 import gen_data, FORMATTER, do_test
-import json
-import os
-import random
-import psycopg2
-import time
-from minio import Minio
-
-
-def upload_to_s3_bucket(config, minio_client, run_id, files, start_bias):
-    _local = lambda idx, start_bias: f"data_{idx + start_bias}.{fmt}"
-    _s3 = lambda idx, start_bias: f"{run_id}_data_{idx + start_bias}.{fmt}"
-    for idx, file_str in enumerate(files):
-        with open(_local(idx, start_bias), "w") as f:
-            f.write(file_str)
-            os.fsync(f.fileno())
-
-        minio_client.fput_object(
-            config["S3_BUCKET"], _s3(idx, start_bias), _local(idx, start_bias)
-        )
-
-
-def check_for_new_files(file_num, item_num_per_file, fmt):
-    conn = psycopg2.connect(host="localhost", port="4566", user="root", database="dev")
-
-    # Open a cursor to execute SQL statements
-    cur = conn.cursor()
-
-    def _table():
-        return f"s3_test_{fmt}"
-
-    total_rows = file_num * item_num_per_file
-
-    MAX_RETRIES = 40
-    for retry_no in range(MAX_RETRIES):
-        cur.execute(f"select count(*) from {_table()}")
-        result = cur.fetchone()
-        if result[0] == total_rows:
-            return True
-        print(
-            f"[retry {retry_no}] Now got {result[0]} rows in table, {total_rows} expected, wait 10s"
-        )
-        time.sleep(10)
-    return False
-
-
-if __name__ == "__main__":
-    FILE_NUM = 101
-    ITEM_NUM_PER_FILE = 2
-    data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
-    fmt = "json"
-
-    split_idx = 51
-    data_batch1 = data[:split_idx]
-    data_batch2 = data[split_idx:]
-
-    config = json.loads(os.environ["S3_SOURCE_TEST_CONF"])
-    client = Minio(
-        config["S3_ENDPOINT"],
-        access_key=config["S3_ACCESS_KEY"],
-        secret_key=config["S3_SECRET_KEY"],
-        secure=True,
-    )
-    run_id = str(random.randint(1000, 9999))
-    print(f"S3 Source New File Test: run ID: {run_id} to bucket {config['S3_BUCKET']}")
-
-    formatted_batch1 = FORMATTER[fmt](data_batch1)
-    upload_to_s3_bucket(config, client, run_id, formatted_batch1, 0)
-
-    # config in do_test that fs source's list interval is 1s
-    do_test(
-        config, len(data_batch1), ITEM_NUM_PER_FILE, run_id, fmt, need_drop_table=False
-    )
-
-    formatted_batch2 = FORMATTER[fmt](data_batch2)
-    upload_to_s3_bucket(config, client, run_id, formatted_batch2, split_idx)
-
-    success_flag = check_for_new_files(FILE_NUM, ITEM_NUM_PER_FILE, fmt)
-    if success_flag:
-        print("Test(add new file) pass")
-    else:
-        print("Test(add new file) fail")
-
-    _s3 = lambda idx, start_bias: f"{run_id}_data_{idx + start_bias}.{fmt}"
-    # clean up s3 files
-    for idx, _ in enumerate(data):
-        client.remove_object(config["S3_BUCKET"], _s3(idx, 0))
-
-    if success_flag == False:
-        exit(1)
diff --git a/e2e_test/s3/posix_fs_source.py b/e2e_test/s3/posix_fs_source.py
deleted file mode 100644
index a7cea46fa496a..0000000000000
--- a/e2e_test/s3/posix_fs_source.py
+++ /dev/null
@@ -1,136 +0,0 @@
-import os
-import sys
-import csv
-import random
-import psycopg2
-import opendal
-
-from time import sleep
-from io import StringIO
-from functools import partial
-
-def gen_data(file_num, item_num_per_file):
-    assert item_num_per_file % 2 == 0, \
-        f'item_num_per_file should be even to ensure sum(mark) == 0: {item_num_per_file}'
-    return [
-        [{
-            'id': file_id * item_num_per_file + item_id,
-            'name': f'{file_id}_{item_id}',
-            'sex': item_id % 2,
-            'mark': (-1) ** (item_id % 2),
-        } for item_id in range(item_num_per_file)]
-        for file_id in range(file_num)
-    ]
-
-def format_csv(data, with_header):
-    csv_files = []
-
-    for file_data in data:
-        ostream = StringIO()
-        writer = csv.DictWriter(ostream, fieldnames=file_data[0].keys())
-        if with_header:
-            writer.writeheader()
-        for item_data in file_data:
-            writer.writerow(item_data)
-        csv_files.append(ostream.getvalue())
-    return csv_files
-
-
-def do_test(file_num, item_num_per_file, prefix, fmt):
-    conn = psycopg2.connect(
-        host="localhost",
-        port="4566",
-        user="root",
-        database="dev"
-    )
-
-    # Open a cursor to execute SQL statements
-    cur = conn.cursor()
-
-    def _table():
-        return f'posix_fs_test_{fmt}'
-
-    def _encode():
-        return f"CSV (delimiter = ',', without_header = {str('without' in fmt).lower()})"
-
-    # Execute a SELECT statement
-    cur.execute(f'''CREATE TABLE {_table()}(
-        id int,
-        name TEXT,
-        sex int,
-        mark int,
-    ) WITH (
-        connector = 'posix_fs',
-        match_pattern = '{prefix}*.{fmt}',
-        posix_fs.root = '/tmp',
-    ) FORMAT PLAIN ENCODE {_encode()};''')
-
-    total_rows = file_num * item_num_per_file
-    MAX_RETRIES = 40
-    for retry_no in range(MAX_RETRIES):
-        cur.execute(f'select count(*) from {_table()}')
-        result = cur.fetchone()
-        if result[0] == total_rows:
-            break
-        print(f"[retry {retry_no}] Now got {result[0]} rows in table, {total_rows} expected, wait 30s")
-        sleep(30)
-
-    stmt = f'select count(*), sum(id), sum(sex), sum(mark) from {_table()}'
-    print(f'Execute {stmt}')
-    cur.execute(stmt)
-    result = cur.fetchone()
-
-    print('Got:', result)
-
-    def _assert_eq(field, got, expect):
-        assert got == expect, f'{field} assertion failed: got {got}, expect {expect}.'
-
-    _assert_eq('count(*)', result[0], total_rows)
-    _assert_eq('sum(id)', result[1], (total_rows - 1) * total_rows / 2)
-    _assert_eq('sum(sex)', result[2], total_rows / 2)
-    _assert_eq('sum(mark)', result[3], 0)
-
-    print('Test pass')
-
-    cur.execute(f'drop table {_table()}')
-    cur.close()
-    conn.close()
-
-
-if __name__ == "__main__":
-    FILE_NUM = 4001
-    ITEM_NUM_PER_FILE = 2
-    data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE)
-
-    fmt = sys.argv[1]
-    FORMATTER = {
-        'csv_with_header': partial(format_csv, with_header=True),
-        'csv_without_header': partial(format_csv, with_header=False),
-    }
-    assert fmt in FORMATTER, f"Unsupported format: {fmt}"
-    formatted_files = FORMATTER[fmt](data)
-
-    run_id = str(random.randint(1000, 9999))
-    _local = lambda idx: f'data_{idx}.{fmt}'
-    _posix = lambda idx: f"{run_id}_data_{idx}.{fmt}"
-    # put local files
-    op = opendal.Operator("fs", root="/tmp")
-
-    print("write file to /tmp")
-    for idx, file_str in enumerate(formatted_files):
-        with open(_local(idx), "w") as f:
-            f.write(file_str)
-            os.fsync(f.fileno())
-        file_name = _posix(idx)
-        file_bytes = file_str.encode('utf-8')
-        op.write(file_name, file_bytes)
-
-    # do test
-    print("do test")
-    do_test(FILE_NUM, ITEM_NUM_PER_FILE, run_id, fmt)
-
-    # clean up local files
-    print("clean up local files in /tmp")
-    for idx, _ in enumerate(formatted_files):
-        file_name = _posix(idx)
-        op.delete(file_name)
diff --git a/e2e_test/sink/kafka/avro.slt b/e2e_test/sink/kafka/avro.slt
index e8a59111cd1b7..28c1f200fbf03 100644
--- a/e2e_test/sink/kafka/avro.slt
+++ b/e2e_test/sink/kafka/avro.slt
@@ -4,11 +4,13 @@ set sink_decouple = false;
 system ok
 rpk topic create test-rw-sink-upsert-avro
 
+# use jq to remove 2 fields/columns before registration
 system ok
-python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-upsert-avro-value' src/connector/src/test_data/all-types.avsc
+jq '.fields = (.fields | map(select(.name | test("unsupported|mon_day_sec_field") | not)))' src/connector/src/test_data/all-types.avsc | sr_register 'test-rw-sink-upsert-avro-value' AVRO
 
+# use jq to select 2 fields/columns used as key
 system ok
-python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-upsert-avro-key' src/connector/src/test_data/all-types.avsc 'string_field,int32_field'
+jq '.fields = (.fields | map(select(.name | test("string_field|int32_field"))))' src/connector/src/test_data/all-types.avsc | sr_register 'test-rw-sink-upsert-avro-key' AVRO
 
 statement ok
 create table from_kafka ( *, gen_i32_field int as int32_field + 2, primary key (some_key) )
@@ -58,7 +60,7 @@ system ok
 rpk topic create test-rw-sink-plain-avro
 
 system ok
-jq '{"schema": tojson}' << EOF | curl -X POST -H 'content-type: application/json' -d @- 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-value/versions'
+sr_register test-rw-sink-plain-avro-value AVRO << EOF
 {
   "type": "record",
   "name": "Simple",
@@ -106,7 +108,7 @@ format plain encode avro (
 key encode text;
 
 system ok
-jq '{"schema": tojson}' << EOF | curl -X POST -H 'content-type: application/json' -d @- 'http://schemaregistry:8082/subjects/test-rw-sink-plain-avro-key/versions'
+sr_register test-rw-sink-plain-avro-key AVRO << EOF
 {
   "type": "record",
   "name": "Key",
diff --git a/e2e_test/sink/kafka/create_sink.slt b/e2e_test/sink/kafka/create_sink.slt
index 338465c471af9..7f589e4a4b231 100644
--- a/e2e_test/sink/kafka/create_sink.slt
+++ b/e2e_test/sink/kafka/create_sink.slt
@@ -31,48 +31,6 @@ create sink sink_non_exist_broker from t_kafka with (
     type = 'append-only',
 );
 
-# Test create sink with connection
-# Create a mock connection
-statement ok
-create connection mock with (
-    type = 'privatelink',
-    provider = 'mock',
-);
-
-# Refer to a non-existant connection
-statement error
-create sink si_kafka_append_only_conn from t_kafka with (
-    connector = 'kafka',
-    properties.bootstrap.server = 'message_queue:29092',
-    topic = 'test-rw-sink-append-only',
-    type = 'append-only',
-    force_append_only = 'true',
-    connection.name = 'nonexist',
-);
-
-# Create sink with connection
-statement ok
-create sink si_kafka_append_only_conn from t_kafka with (
-    connector = 'kafka',
-    properties.bootstrap.server = 'message_queue:29092',
-    topic = 'test-rw-sink-append-only',
-    type = 'append-only',
-    force_append_only = 'true',
-    connection.name = 'mock',
-);
-
-# Try to drop connection mock, which is in use
-statement error
-drop connection mock;
-
-# Drop sink
-statement ok
-drop sink si_kafka_append_only_conn;
-
-# Drop connection
-statement ok
-drop connection mock;
-
 # Connection test clean-up finished
 
 statement error sink cannot be append-only
diff --git a/e2e_test/sink/kafka/protobuf.slt b/e2e_test/sink/kafka/protobuf.slt
index 25b95a49cf1f3..82d1a0693f24f 100644
--- a/e2e_test/sink/kafka/protobuf.slt
+++ b/e2e_test/sink/kafka/protobuf.slt
@@ -17,7 +17,7 @@ system ok
 rpk topic create test-rw-sink-append-only-protobuf-csr-a
 
 system ok
-python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-append-only-protobuf-csr-a-value' src/connector/src/test_data/test-index-array.proto
+sr_register 'test-rw-sink-append-only-protobuf-csr-a-value' PROTOBUF < src/connector/src/test_data/test-index-array.proto
 
 statement ok
 create table from_kafka_csr_trivial with (
@@ -32,7 +32,7 @@ system ok
 rpk topic create test-rw-sink-append-only-protobuf-csr-hi
 
 system ok
-python3 e2e_test/sink/kafka/register_schema.py 'http://schemaregistry:8082' 'test-rw-sink-append-only-protobuf-csr-hi-value' src/connector/src/test_data/test-index-array.proto
+sr_register 'test-rw-sink-append-only-protobuf-csr-hi-value' PROTOBUF < src/connector/src/test_data/test-index-array.proto
 
 statement ok
 create table from_kafka_csr_nested with (
diff --git a/e2e_test/sink/kafka/register_schema.py b/e2e_test/sink/kafka/register_schema.py
deleted file mode 100644
index 7385725ca8d34..0000000000000
--- a/e2e_test/sink/kafka/register_schema.py
+++ /dev/null
@@ -1,54 +0,0 @@
-import sys
-from confluent_kafka.schema_registry import SchemaRegistryClient, Schema
-
-
-def main():
-    url = sys.argv[1]
-    subject = sys.argv[2]
-    local_path = sys.argv[3]
-    with open(local_path) as f:
-        schema_str = f.read()
-    if 4 < len(sys.argv):
-        keys = sys.argv[4].split(',')
-    else:
-        keys = []
-
-    client = SchemaRegistryClient({"url": url})
-
-    if local_path.endswith('.avsc'):
-        if keys:
-            schema_str = select_keys(schema_str, keys)
-        else:
-            schema_str = remove_unsupported(schema_str)
-        schema = Schema(schema_str, 'AVRO')
-    elif local_path.endswith('.proto'):
-        schema = Schema(schema_str, 'PROTOBUF')
-    else:
-        raise ValueError('{} shall end with .avsc or .proto'.format(local_path))
-    client.register_schema(subject, schema)
-
-
-def select_fields(schema_str, f):
-    import json
-    root = json.loads(schema_str)
-    if not isinstance(root, dict):
-        return schema_str
-    if root['type'] != 'record':
-        return schema_str
-    root['fields'] = f(root['fields'])
-    return json.dumps(root)
-
-
-def remove_unsupported(schema_str):
-    return select_fields(schema_str, lambda fields: [f for f in fields if f['name'] not in {'unsupported', 'mon_day_sec_field'}])
-
-
-def select_keys(schema_str, keys):
-    def process(fields):
-        by_name = {f['name']: f for f in fields}
-        return [by_name[k] for k in keys]
-    return select_fields(schema_str, process)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/e2e_test/sink/mqtt_sink.slt b/e2e_test/sink/mqtt_sink.slt
index 2602d2ddc6198..cdf0f1d5233f4 100644
--- a/e2e_test/sink/mqtt_sink.slt
+++ b/e2e_test/sink/mqtt_sink.slt
@@ -47,6 +47,34 @@ WITH
       force_append_only='true',
   );
 
+# First the (retained) topics are primed, so that they will be listened
+# to when the mqtt source initializes. Otherwise it would take 30 seconds
+# for the next enumerator tick
+
+statement ok
+INSERT INTO mqtt (device_id, temperature)
+VALUES ( '12',  56.0 );
+
+statement ok
+FLUSH;
+
+statement ok
+INSERT INTO mqtt (device_id, temperature)
+VALUES ( '13',  20.0 );
+
+statement ok
+FLUSH;
+
+statement ok
+INSERT INTO mqtt_nested (info, temperature)
+VALUES( ROW('12', '/nested/12'), 56.0 );
+
+statement ok
+FLUSH;
+
+statement ok
+INSERT INTO mqtt_nested (info, temperature)
+VALUES( ROW('13', null), 22.0 );
 
 statement ok
 CREATE TABLE mqtt_source
@@ -54,10 +82,11 @@ CREATE TABLE mqtt_source
   device_id varchar,
   temperature double
 )
+INCLUDE partition AS mqtt_topic
 WITH (
-    connector='mqtt',
-    url='tcp://mqtt-server',
-    topic= '/device/+',
+    connector ='mqtt',
+    url ='tcp://mqtt-server',
+    topic = '/device/+',
     qos = 'at_least_once',
 ) FORMAT PLAIN ENCODE JSON;
 
@@ -75,29 +104,23 @@ WITH (
 ) FORMAT PLAIN ENCODE JSON;
 
 
-statement ok
-INSERT INTO mqtt (device_id, temperature)
-VALUES ( '12',  56.0 );
-
 statement ok
 INSERT INTO mqtt (device_id, temperature)
 VALUES ( '12',  59.0 );
 
 statement ok
-INSERT INTO mqtt (device_id, temperature)
-VALUES ( '13',  20.0 );
+FLUSH;
 
 statement ok
 INSERT INTO mqtt (device_id, temperature)
 VALUES ( '13',  22.0 );
 
 statement ok
-INSERT INTO mqtt_nested (info, temperature)
-VALUES( ROW('12', '/nested/12'), 56.0 );
+FLUSH;
 
 statement ok
 INSERT INTO mqtt_nested (info, temperature)
-VALUES( ROW('13', null), 22.0 );
+VALUES( ROW('12', '/nested/12'), 56.0 );
 
 statement ok
 FLUSH;
@@ -112,6 +135,14 @@ SELECT device_id, temperature FROM mqtt ORDER BY device_id, temperature;
 13 20
 13 22
 
+query ITT rowsort
+SELECT device_id, temperature, mqtt_topic FROM mqtt_source ORDER BY device_id, temperature;
+----
+12 56 /device/12
+12 59 /device/12
+13 20 /device/13
+13 22 /device/13
+
 query IT rowsort
 SELECT (info).device_id device_id, temperature from mqtt_nested_source ORDER BY device_id, temperature ;
 ----
diff --git a/e2e_test/sink/sink_into_table/snapshot.slt b/e2e_test/sink/sink_into_table/snapshot.slt
new file mode 100644
index 0000000000000..461448098b498
--- /dev/null
+++ b/e2e_test/sink/sink_into_table/snapshot.slt
@@ -0,0 +1,40 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+create table s (v1 int, v2 int) APPEND ONLY;
+
+statement ok
+insert into s values (1, 11);
+
+statement ok
+create table t (v1 int, v2 int);
+
+statement ok
+create sink s1 into t from s with (type = 'append-only');
+
+statement ok
+create sink s2 into t from s with (type = 'append-only', snapshot = 'false');
+
+statement ok
+insert into s values (2, 22);
+
+
+query II rowsort
+select * from t;
+----
+1	11
+2	22
+2	22
+
+statement ok
+drop sink s1;
+
+statement ok
+drop sink s2;
+
+statement ok
+drop table t;
+
+statement ok
+drop table s;
diff --git a/e2e_test/source_inline/README.md b/e2e_test/source_inline/README.md
index 8f766c5637fb6..fa8cf25c56b0f 100644
--- a/e2e_test/source_inline/README.md
+++ b/e2e_test/source_inline/README.md
@@ -5,6 +5,19 @@ Compared with prior source tests ( `e2e_test/source` ), tests in this directory
 See the [connector development guide](http://risingwavelabs.github.io/risingwave/connector/intro.html#end-to-end-tests) for more information about how to set up the test environment,
 run tests, and write tests.
 
+## Serial Tests
+
+Tests ending with `.slt.serial` can only be run in serial, e.g., it contains `RECOVER` statement which will break other connections, or it has some special `system` commands.
+
+Other tests can be run in parallel.
+
+```bash
+# run all parallel tests
+risedev slt './e2e_test/source_inline/**/*.slt' -j16
+# run all serial tests
+risedev slt './e2e_test/source_inline/**/*.slt.serial'
+```
+
 ## Install Dependencies
 
 Some additional tools are needed to run the `system` commands in tests.
diff --git a/e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt b/e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt.serial
similarity index 99%
rename from e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt
rename to e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt.serial
index 10854d97b6440..fde008079dc6e 100644
--- a/e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt
+++ b/e2e_test/source_inline/cdc/mysql/mysql_create_drop.slt.serial
@@ -45,7 +45,6 @@ create source s with (
   username = 'shared-cdc',
   password = 'abcdef',
   database.name = 'testdb1',
-  server.id = '114514'
 );
 
 sleep 2s
diff --git a/e2e_test/source_inline/kafka/alter/add_column.slt b/e2e_test/source_inline/kafka/alter/add_column.slt
index cde818f6baf90..9d0bbe671079d 100644
--- a/e2e_test/source_inline/kafka/alter/add_column.slt
+++ b/e2e_test/source_inline/kafka/alter/add_column.slt
@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 system ok
 rpk topic delete kafka_alter || true
 
@@ -269,3 +272,6 @@ select * from t
 
 statement ok
 drop table t;
+
+statement ok
+SET streaming_use_shared_source TO true;
diff --git a/e2e_test/source_inline/kafka/alter/rate_limit_source_kafka.slt b/e2e_test/source_inline/kafka/alter/rate_limit_source_kafka.slt
index 9bfa7238e3476..96fd016c5812d 100644
--- a/e2e_test/source_inline/kafka/alter/rate_limit_source_kafka.slt
+++ b/e2e_test/source_inline/kafka/alter/rate_limit_source_kafka.slt
@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 ############## Create kafka seed data
 
 statement ok
@@ -127,3 +130,6 @@ drop sink kafka_sink;
 
 statement ok
 drop table kafka_seed_data;
+
+statement ok
+SET streaming_use_shared_source TO true;
diff --git a/e2e_test/source_inline/kafka/avro/alter_source.slt b/e2e_test/source_inline/kafka/avro/alter_source.slt
index 57677af57cd92..8bce7f4efd5cf 100644
--- a/e2e_test/source_inline/kafka/avro/alter_source.slt
+++ b/e2e_test/source_inline/kafka/avro/alter_source.slt
@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 # https://github.com/risingwavelabs/risingwave/issues/16486
 
 # cleanup
@@ -12,7 +15,7 @@ system ok
 rpk topic create 'avro_alter_source_test'
 
 system ok
-sr_register avro_alter_source_test-value '{"type":"record","name":"Root","fields":[{"name":"foo","type":"string"}]}'
+sr_register avro_alter_source_test-value AVRO <<< '{"type":"record","name":"Root","fields":[{"name":"foo","type":"string"}]}'
 
 statement ok
 create source s
@@ -26,7 +29,7 @@ FORMAT PLAIN ENCODE AVRO (
 
 # create a new version of schema and produce a message
 system ok
-sr_register avro_alter_source_test-value '{"type":"record","name":"Root","fields":[{"name":"bar","type":"int","default":0},{"name":"foo","type":"string"}]}'
+sr_register avro_alter_source_test-value AVRO <<< '{"type":"record","name":"Root","fields":[{"name":"bar","type":"int","default":0},{"name":"foo","type":"string"}]}'
 
 system ok
 echo '{"foo":"ABC", "bar":1}' | rpk topic produce --schema-id=topic avro_alter_source_test
@@ -66,3 +69,6 @@ ABC	1
 
 statement ok
 drop source s cascade;
+
+statement ok
+SET streaming_use_shared_source TO true;
diff --git a/e2e_test/source_inline/kafka/avro/alter_table.slt b/e2e_test/source_inline/kafka/avro/alter_table.slt
index e8c43739b4746..330cdc490cdba 100644
--- a/e2e_test/source_inline/kafka/avro/alter_table.slt
+++ b/e2e_test/source_inline/kafka/avro/alter_table.slt
@@ -13,8 +13,7 @@ rpk topic create 'avro_alter_table_test'
 
 # create a schema and produce a message
 system ok
-echo '{"type":"record","name":"Root","fields":[{"name":"bar","type":"int","default":0},{"name":"foo","type":"string"}]}' | jq '{"schema": tojson}' \
-| curl -s -X POST -H 'content-type:application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/avro_alter_table_test-value/versions"
+sr_register avro_alter_table_test-value AVRO <<< '{"type":"record","name":"Root","fields":[{"name":"bar","type":"int","default":0},{"name":"foo","type":"string"}]}'
 
 system ok
 echo '{"foo":"ABC", "bar":1}' | rpk topic produce --schema-id=topic avro_alter_table_test
@@ -38,8 +37,7 @@ select * from t
 
 # create a new version of schema that removed field bar
 system ok
-echo '{"type":"record","name":"Root","fields":[{"name":"foo","type":"string"}]}' | jq '{"schema": tojson}' \
-| curl -s -X POST -H 'content-type:application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/avro_alter_table_test-value/versions"
+sr_register avro_alter_table_test-value AVRO <<< '{"type":"record","name":"Root","fields":[{"name":"foo","type":"string"}]}'
 
 # Refresh table schema should fail
 statement error
diff --git a/e2e_test/source_inline/kafka/avro/glue.slt b/e2e_test/source_inline/kafka/avro/glue.slt
index 4be9378b530df..a4359e9a7c124 100644
--- a/e2e_test/source_inline/kafka/avro/glue.slt
+++ b/e2e_test/source_inline/kafka/avro/glue.slt
@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 system ok
 rpk topic delete 'glue-sample-my-event'
 
@@ -14,6 +17,8 @@ EOF
 statement ok
 ALTER SYSTEM SET license_key TO '';
 
+sleep 1s
+
 statement error
 create source t with (
   connector = 'kafka',
@@ -146,3 +151,6 @@ drop source t;
 
 system ok
 rpk topic delete 'glue-sample-my-event'
+
+statement ok
+SET streaming_use_shared_source TO true;
diff --git a/e2e_test/source_inline/kafka/avro/union.slt b/e2e_test/source_inline/kafka/avro/union.slt
index 44e1db659d120..52dd55f856cc9 100644
--- a/e2e_test/source_inline/kafka/avro/union.slt
+++ b/e2e_test/source_inline/kafka/avro/union.slt
@@ -6,7 +6,7 @@ rpk topic delete 'avro-union' || true; \
 rpk topic create avro-union
 
 system ok
-sr_register avro-union-value '
+sr_register avro-union-value AVRO <<EOF
 {
   "type": "record",
   "name": "Root",
@@ -36,7 +36,7 @@ sr_register avro-union-value '
     }
   ]
 }
-'
+EOF
 
 system ok
 cat<<EOF | rpk topic produce avro-union --schema-id=topic
@@ -107,7 +107,7 @@ rpk topic delete 'avro-union-simple' || true; \
 rpk topic create avro-union-simple
 
 system ok
-sr_register avro-union-simple-value '
+sr_register avro-union-simple-value AVRO <<EOF
 {
   "type": "record",
   "name": "Root",
@@ -118,7 +118,7 @@ sr_register avro-union-simple-value '
     }
   ]
 }
-'
+EOF
 
 system ok
 cat<<EOF | rpk topic produce avro-union-simple --schema-id=topic
diff --git a/e2e_test/source_inline/kafka/background_ddl/basic.slt.serial b/e2e_test/source_inline/kafka/background_ddl/basic.slt.serial
new file mode 100644
index 0000000000000..75a7601661a89
--- /dev/null
+++ b/e2e_test/source_inline/kafka/background_ddl/basic.slt.serial
@@ -0,0 +1,130 @@
+control substitution on
+
+statement ok
+SET streaming_use_shared_source TO false;
+
+############## Create kafka seed data
+
+statement ok
+create table kafka_seed_data (v1 int);
+
+statement ok
+insert into kafka_seed_data select * from generate_series(1, 1000);
+
+statement ok
+create sink kafka_sink
+from
+  kafka_seed_data with (
+    ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+    topic = 'test_rate_limit',
+    type = 'append-only',
+    force_append_only='true'
+);
+
+# Wait for the topic to create
+skipif in-memory
+sleep 5s
+
+############## Create kafka source
+
+statement ok
+create source kafka_source (v1 int) with (
+  ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+  topic = 'test_rate_limit',
+  scan.startup.mode = 'earliest',
+  source_rate_limit = 1,
+) FORMAT PLAIN ENCODE JSON
+
+statement ok
+flush;
+
+############## Create background mv with only source upstream
+
+statement ok
+set background_ddl = true;
+
+statement ok
+create materialized view rl_mv1 as select count(*) from kafka_source;
+
+statement ok
+wait
+
+statement ok
+select * from rl_mv1;
+
+statement ok
+drop materialized view rl_mv1;
+
+############## Create background mv joined with another mv
+
+statement ok
+set streaming_parallelism=1;
+
+statement ok
+create table t1(v1 int);
+
+statement ok
+insert into t1 select * from generate_series(1, 10);
+
+statement ok
+flush;
+
+statement ok
+set backfill_rate_limit = 1;
+
+statement ok
+set background_ddl=true;
+
+statement ok
+create materialized view rl_mv2 as select kafka_source.v1 from kafka_source join t1 on kafka_source.v1 = t1.v1;
+
+query I
+select count(*) from rw_catalog.rw_ddl_progress;
+----
+1
+
+sleep 1s
+
+statement ok
+recover
+
+sleep 5s
+
+query I
+select count(*) from rw_catalog.rw_ddl_progress;
+----
+1
+
+statement ok
+wait
+
+query I
+select count(*) from rw_catalog.rw_ddl_progress;
+----
+0
+
+statement ok
+drop materialized view rl_mv2;
+
+statement ok
+drop table t1;
+
+############## Cleanup
+
+statement ok
+drop source kafka_source;
+
+statement ok
+drop sink kafka_sink;
+
+statement ok
+drop table kafka_seed_data;
+
+statement ok
+set background_ddl=false;
+
+statement ok
+SET streaming_use_shared_source TO true;
+
+statement ok
+set streaming_parallelism=default;
diff --git a/e2e_test/source_inline/kafka/consumer_group.slt b/e2e_test/source_inline/kafka/consumer_group.slt.serial
similarity index 100%
rename from e2e_test/source_inline/kafka/consumer_group.slt
rename to e2e_test/source_inline/kafka/consumer_group.slt.serial
diff --git a/e2e_test/source_inline/kafka/protobuf/alter_source.slt b/e2e_test/source_inline/kafka/protobuf/alter_source.slt
index e94ffc9541504..7f1b431a06bac 100644
--- a/e2e_test/source_inline/kafka/protobuf/alter_source.slt
+++ b/e2e_test/source_inline/kafka/protobuf/alter_source.slt
@@ -1,18 +1,21 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 system ok
-rpk topic delete sr_pb_test || true; \
-(rpk sr subject delete 'sr_pb_test-value' && rpk sr subject delete 'sr_pb_test-value' --permanent) || true;
+rpk topic delete pb_alter_source_test || true; \
+(rpk sr subject delete 'pb_alter_source_test-value' && rpk sr subject delete 'pb_alter_source_test-value' --permanent) || true;
 
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 20 user
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 20 user
 
 statement ok
 CREATE SOURCE src_user
 INCLUDE timestamp -- include explicitly here to test a bug found in https://github.com/risingwavelabs/risingwave/pull/17293
 WITH (
     ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
-    topic = 'sr_pb_test',
+    topic = 'pb_alter_source_test',
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
@@ -26,7 +29,7 @@ CREATE MATERIALIZED VIEW mv_user AS SELECT * FROM src_user;
 statement ok
 CREATE TABLE t_user WITH (
     ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
-    topic = 'sr_pb_test',
+    topic = 'pb_alter_source_test',
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
@@ -42,7 +45,7 @@ SELECT age FROM t_user;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 5 user_with_more_fields
 
 sleep 5s
 
@@ -69,7 +72,7 @@ SELECT COUNT(*), MAX(age), MIN(age), SUM(age) FROM t_user;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 5 user_with_more_fields
 
 sleep 5s
 
@@ -89,3 +92,6 @@ DROP MATERIALIZED VIEW mv_user;
 
 statement ok
 DROP SOURCE src_user;
+
+statement ok
+SET streaming_use_shared_source TO true;
diff --git a/e2e_test/source_inline/kafka/protobuf/recover.slt b/e2e_test/source_inline/kafka/protobuf/recover.slt.serial
similarity index 80%
rename from e2e_test/source_inline/kafka/protobuf/recover.slt
rename to e2e_test/source_inline/kafka/protobuf/recover.slt.serial
index 3babf26793f2a..6cf786f0da4df 100644
--- a/e2e_test/source_inline/kafka/protobuf/recover.slt
+++ b/e2e_test/source_inline/kafka/protobuf/recover.slt.serial
@@ -5,7 +5,7 @@ rpk topic create 'test-pb-struct'
 
 
 system ok
-jq -sR '{"schema":.,"schemaType":"PROTOBUF"}' << EOF | curl -X POST -H 'content-type: application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/test-pb-struct-value/versions"
+sr_register test-pb-struct-value PROTOBUF << EOF
 syntax = "proto3";
 package test;
 message User {
@@ -31,7 +31,7 @@ format plain encode protobuf (
 
 # register a v2 schema
 system ok
-jq -sR '{"schema":.,"schemaType":"PROTOBUF"}' << EOF | curl -X POST -H 'content-type: application/json' -d @- "${RISEDEV_SCHEMA_REGISTRY_URL}/subjects/test-pb-struct-value/versions"
+sr_register test-pb-struct-value PROTOBUF << EOF
 syntax = "proto3";
 package test;
 message User {
diff --git a/e2e_test/source_inline/kafka/secret_dep.slt b/e2e_test/source_inline/kafka/secret_dep.slt
new file mode 100644
index 0000000000000..93426bdf5073b
--- /dev/null
+++ b/e2e_test/source_inline/kafka/secret_dep.slt
@@ -0,0 +1,32 @@
+# Note: control substitution on will force us to use "\n" instead of "\n" in commands
+control substitution on
+
+# for non-shared source
+statement ok
+set streaming_use_shared_source to false;
+
+system ok
+rpk topic create test_secret_ref -p 3
+
+statement ok
+CREATE SECRET sec WITH (backend = 'meta') AS 'test_secret_ref';
+
+statement ok
+CREATE SOURCE s(x varchar)
+WITH(
+    ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
+    topic = secret sec,
+    scan.startup.mode = 'earliest',
+) FORMAT PLAIN ENCODE JSON;
+
+statement error
+DROP SECRET sec;
+
+statement ok
+DROP SOURCE s;
+
+statement ok
+DROP SECRET sec;
+
+statement ok
+set streaming_use_shared_source to true;
diff --git a/e2e_test/source_inline/kafka/shared_source.slt b/e2e_test/source_inline/kafka/shared_source.slt.serial
similarity index 79%
rename from e2e_test/source_inline/kafka/shared_source.slt
rename to e2e_test/source_inline/kafka/shared_source.slt.serial
index ca429781604e8..3397f90f081da 100644
--- a/e2e_test/source_inline/kafka/shared_source.slt
+++ b/e2e_test/source_inline/kafka/shared_source.slt.serial
@@ -1,7 +1,7 @@
 control substitution on
 
 statement ok
-SET enable_shared_source TO true;
+SET streaming_use_shared_source TO true;
 
 system ok
 rpk topic create shared_source -p 4
@@ -14,6 +14,11 @@ create source s_before_produce (v1 int, v2 varchar) with (
   scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE JSON;
 
+query T
+select connector, is_shared from rw_sources where name = 's_before_produce';
+----
+KAFKA t
+
 statement ok
 create materialized view mv_before_produce as select * from s_before_produce;
 
@@ -23,10 +28,10 @@ sleep 2s
 system ok
 internal_table.mjs --name mv_before_produce --type sourcebackfill
 ----
-0,"""Finished"""
-1,"""Finished"""
-2,"""Finished"""
-3,"""Finished"""
+0,"{""num_consumed_rows"": 0, ""state"": ""Finished"", ""target_offset"": null}"
+1,"{""num_consumed_rows"": 0, ""state"": ""Finished"", ""target_offset"": null}"
+2,"{""num_consumed_rows"": 0, ""state"": ""Finished"", ""target_offset"": null}"
+3,"{""num_consumed_rows"": 0, ""state"": ""Finished"", ""target_offset"": null}"
 
 
 system ok
@@ -34,6 +39,9 @@ cat << EOF | rpk topic produce shared_source -f "%p %v\n" -p 0
 0 {"v1": 1, "v2": "a"}
 1 {"v1": 2, "v2": "b"}
 2 {"v1": 3, "v2": "c"}
+2 {"v1": 3, "v2": "c"}
+3 {"v1": 4, "v2": "d"}
+3 {"v1": 4, "v2": "d"}
 3 {"v1": 4, "v2": "d"}
 EOF
 
@@ -78,15 +86,15 @@ internal_table.mjs --name s0 --type source
 system ok
 internal_table.mjs --name mv_1 --type sourcebackfill
 ----
-0,"{""SourceCachingUp"": ""0""}"
-1,"{""SourceCachingUp"": ""0""}"
-2,"{""SourceCachingUp"": ""0""}"
-3,"{""SourceCachingUp"": ""0""}"
+0,"{""num_consumed_rows"": 1, ""state"": {""SourceCachingUp"": ""0""}, ""target_offset"": ""0""}"
+1,"{""num_consumed_rows"": 1, ""state"": {""SourceCachingUp"": ""0""}, ""target_offset"": ""0""}"
+2,"{""num_consumed_rows"": 2, ""state"": {""SourceCachingUp"": ""1""}, ""target_offset"": ""1""}"
+3,"{""num_consumed_rows"": 3, ""state"": {""SourceCachingUp"": ""2""}, ""target_offset"": ""2""}"
 
 
 # This does not affect the behavior for CREATE MATERIALIZED VIEW below. It also uses the shared source, and creates SourceBackfillExecutor.
 statement ok
-SET enable_shared_source TO false;
+SET streaming_use_shared_source TO false;
 
 statement ok
 create materialized view mv_2 as select * from s0;
@@ -96,26 +104,35 @@ sleep 2s
 query ?? rowsort
 select v1, v2 from s0;
 ----
-1 a
-2 b
-3 c
-4 d
+1	a
+2	b
+3	c
+3	c
+4	d
+4	d
+4	d
 
 query ?? rowsort
 select v1, v2 from mv_1;
 ----
-1 a
-2 b
-3 c
-4 d
+1	a
+2	b
+3	c
+3	c
+4	d
+4	d
+4	d
 
 query ?? rowsort
 select v1, v2 from mv_2;
 ----
-1 a
-2 b
-3 c
-4 d
+1	a
+2	b
+3	c
+3	c
+4	d
+4	d
+4	d
 
 system ok
 cat << EOF | rpk topic produce shared_source -f "%p %v\n" -p 0
@@ -133,33 +150,39 @@ internal_table.mjs --name s0 --type source
 ----
 0,"{""split_info"": {""partition"": 0, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
 1,"{""split_info"": {""partition"": 1, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
-2,"{""split_info"": {""partition"": 2, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
-3,"{""split_info"": {""partition"": 3, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+2,"{""split_info"": {""partition"": 2, ""start_offset"": 2, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+3,"{""split_info"": {""partition"": 3, ""start_offset"": 3, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
 
 
 query ?? rowsort
 select v1, v2 from s0;
 ----
-1 a
-1 aa
-2 b
-2 bb
-3 c
-3 cc
-4 d
-4 dd
+1	a
+1	aa
+2	b
+2	bb
+3	c
+3	c
+3	cc
+4	d
+4	d
+4	d
+4	dd
 
 query ?? rowsort
 select v1, v2 from mv_1;
 ----
-1 a
-1 aa
-2 b
-2 bb
-3 c
-3 cc
-4 d
-4 dd
+1	a
+1	aa
+2	b
+2	bb
+3	c
+3	c
+3	cc
+4	d
+4	d
+4	d
+4	dd
 
 
 # start_offset changed to 1
@@ -168,18 +191,18 @@ internal_table.mjs --name s0 --type source
 ----
 0,"{""split_info"": {""partition"": 0, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
 1,"{""split_info"": {""partition"": 1, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
-2,"{""split_info"": {""partition"": 2, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
-3,"{""split_info"": {""partition"": 3, ""start_offset"": 1, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+2,"{""split_info"": {""partition"": 2, ""start_offset"": 2, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+3,"{""split_info"": {""partition"": 3, ""start_offset"": 3, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
 
 
 # Transition from SourceCachingUp to Finished after consuming one upstream message.
 system ok
 internal_table.mjs --name mv_1 --type sourcebackfill
 ----
-0,"""Finished"""
-1,"""Finished"""
-2,"""Finished"""
-3,"""Finished"""
+0,"{""num_consumed_rows"": 2, ""state"": ""Finished"", ""target_offset"": ""0""}"
+1,"{""num_consumed_rows"": 2, ""state"": ""Finished"", ""target_offset"": ""0""}"
+2,"{""num_consumed_rows"": 3, ""state"": ""Finished"", ""target_offset"": ""1""}"
+3,"{""num_consumed_rows"": 4, ""state"": ""Finished"", ""target_offset"": ""2""}"
 
 
 system ok
@@ -198,26 +221,26 @@ sleep 3s
 query ?? rowsort
 select v1, count(*) from s0 group by v1;
 ----
-1 12
-2 12
-3 12
-4 12
+1	12
+2	12
+3	13
+4	14
 
 query ?? rowsort
 select v1, count(*) from mv_1 group by v1;
 ----
-1 12
-2 12
-3 12
-4 12
+1	12
+2	12
+3	13
+4	14
 
 query ?? rowsort
 select v1, count(*) from mv_before_produce group by v1;
 ----
-1 12
-2 12
-3 12
-4 12
+1	12
+2	12
+3	13
+4	14
 
 
 # start_offset changed to 11
@@ -226,8 +249,8 @@ internal_table.mjs --name s0 --type source
 ----
 0,"{""split_info"": {""partition"": 0, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
 1,"{""split_info"": {""partition"": 1, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
-2,"{""split_info"": {""partition"": 2, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
-3,"{""split_info"": {""partition"": 3, ""start_offset"": 11, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+2,"{""split_info"": {""partition"": 2, ""start_offset"": 12, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
+3,"{""split_info"": {""partition"": 3, ""start_offset"": 13, ""stop_offset"": null, ""topic"": ""shared_source""}, ""split_type"": ""kafka""}"
 
 
 # # Note: the parallelism depends on the risedev profile.
diff --git a/e2e_test/source_inline/kafka/temporary_kafka_batch.slt b/e2e_test/source_inline/kafka/temporary_kafka_batch.slt
index 69144a884486f..47c6124e595d0 100644
--- a/e2e_test/source_inline/kafka/temporary_kafka_batch.slt
+++ b/e2e_test/source_inline/kafka/temporary_kafka_batch.slt
@@ -71,6 +71,12 @@ t
 t
 t
 
+# ensure now() is larger
+sleep 1s
+
+statement ok
+flush;
+
 query B
 select _rw_kafka_timestamp < now() from s1
 ----
diff --git a/e2e_test/source_inline/pubsub/pubsub.slt b/e2e_test/source_inline/pubsub/pubsub.slt.serial
similarity index 100%
rename from e2e_test/source_inline/pubsub/pubsub.slt
rename to e2e_test/source_inline/pubsub/pubsub.slt.serial
diff --git a/e2e_test/source_inline/tvf/mysql_query.slt b/e2e_test/source_inline/tvf/mysql_query.slt
new file mode 100644
index 0000000000000..56acf0598244c
--- /dev/null
+++ b/e2e_test/source_inline/tvf/mysql_query.slt
@@ -0,0 +1,73 @@
+control substitution on
+
+system ok
+mysql -e "DROP DATABASE IF EXISTS tvf; CREATE DATABASE tvf;"
+
+system ok
+mysql -e "
+USE tvf;
+CREATE TABLE test (
+    id bigint primary key,
+    v0 bit,
+    v1 bool,
+    v2 tinyint(1),
+    v3 tinyint(2),
+    v4 smallint,
+    v5 mediumint,
+    v6 integer,
+    v7 bigint,
+    v8 float,
+    v9 double,
+    v10 numeric(4, 2),
+    v11 decimal(4, 2),
+    v12 char(255),
+    v13 varchar(255),
+    v14 bit(10),
+    v15 tinyblob,
+    v16 blob,
+    v17 mediumblob,
+    v18 longblob,
+    v19 date,
+    v20 time,
+    v21 timestamp,
+    v22 json,
+    v23 int
+);
+INSERT INTO test SELECT
+  1 as id,
+  true as v0,
+  true as v1,
+  2 as v2,
+  3 as v3,
+  4 as v4,
+  5 as v5,
+  6 as v6,
+  7 as v7,
+  1.08 as v8,
+  1.09 as v9,
+  1.10 as v10,
+  1.11 as v11,
+  'char' as v12,
+  'varchar' as v13,
+  b'1010' as v14,
+  x'16' as v15,
+  x'17' as v16,
+  x'18' as v17,
+  x'19' as v18,
+  '2021-01-01' as v19,
+  '12:34:56' as v20,
+  '2021-01-01 12:34:56' as v21,
+  JSON_OBJECT('key1', 1, 'key2', 'abc') as v22,
+  null as v23;
+"
+
+query
+select * from mysql_query('$MYSQL_HOST', '$MYSQL_TCP_PORT', '$RISEDEV_MYSQL_USER', '$MYSQL_PWD', 'tvf', 'select * from test;');
+----
+1 t 1 2 3 4 5 6 7 1.08 1.09 1.10 1.11 char varchar \x000a \x16 \x17 \x18 \x19 2021-01-01 12:34:56 2021-01-01 12:34:56+00:00 {"key1": 1, "key2": "abc"} NULL
+
+system ok
+mysql -e "
+USE tvf;
+DROP DATABASE tvf;
+"
\ No newline at end of file
diff --git a/e2e_test/source_legacy/tvf/postgres_query.slt b/e2e_test/source_inline/tvf/postgres_query.slt
similarity index 85%
rename from e2e_test/source_legacy/tvf/postgres_query.slt
rename to e2e_test/source_inline/tvf/postgres_query.slt
index 0ce577535b923..88e508e442474 100644
--- a/e2e_test/source_legacy/tvf/postgres_query.slt
+++ b/e2e_test/source_inline/tvf/postgres_query.slt
@@ -1,5 +1,10 @@
+control substitution on
+
+system ok
+PGDATABASE=source_test createdb
+
 system ok
-PGHOST=db PGPORT=5432 PGUSER=postgres PGPASSWORD=postgres PGDATABASE=cdc_test psql -c "
+PGDATABASE=source_test psql -c "
 CREATE TABLE test (
     id bigint primary key,
     v1 bool,
@@ -21,12 +26,12 @@ CREATE TABLE test (
 );"
 
 system ok
-PGHOST=db PGPORT=5432 PGUSER=postgres PGPASSWORD=postgres PGDATABASE=cdc_test psql -c "
+PGDATABASE=source_test psql -c "
 INSERT INTO test SELECT generate_series(1, 100), true, 1, 1, 1, 1.0, 1.0, 1.0, '2021-01-01', '00:00:00', '2021-01-01 00:00:00', '2021-01-01 00:00:00 pst', 'text', 'varchar', '1 day', '{}', '\x01';
 "
 
 query II
-select * from postgres_query('db', '5432', 'postgres', 'postgres', 'cdc_test', 'select * from test where id > 90;');
+select * from postgres_query('$PGHOST', '$PGPORT', '$PGUSER', '$PGPASSWORD', 'source_test', 'select * from test where id > 90;');
 ----
 91 t 1 1 1 1 1 1.0 2021-01-01 00:00:00 2021-01-01 00:00:00 2021-01-01 08:00:00+00:00 text varchar 1 day {} \x01
 92 t 1 1 1 1 1 1.0 2021-01-01 00:00:00 2021-01-01 00:00:00 2021-01-01 08:00:00+00:00 text varchar 1 day {} \x01
diff --git a/e2e_test/source_legacy/basic/ddl.slt b/e2e_test/source_legacy/basic/ddl.slt
index 8e63971fb5b82..edf2ea3d9bdc0 100644
--- a/e2e_test/source_legacy/basic/ddl.slt
+++ b/e2e_test/source_legacy/basic/ddl.slt
@@ -30,7 +30,7 @@ Caused by these errors (recent errors listed first):
   1: gRPC request to meta service failed: Internal error
   2: failed to create source worker
   3: failed to parse json
-  4: missing field `properties.bootstrap.server`
+  4: missing field `topic`
 
 
 statement error
@@ -198,35 +198,6 @@ s
 statement ok
 drop table s
 
-# Test create source with connection
-statement ok
-CREATE CONNECTION mock WITH (type = 'privatelink', provider = 'mock');
-
-# Reference to non-existant connection
-statement error
-create source s (
-  column1 varchar
-) with (
-  connector = 'kafka',
-  topic = 'kafka_1_partition_topic',
-  properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
-) FORMAT PLAIN ENCODE JSON;
-
-statement ok
-CREATE TABLE mytable (
-  column1 varchar
-) with (
-  connector = 'kafka',
-  topic = 'kafka_1_partition_topic',
-  properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'mock'
-) FORMAT PLAIN ENCODE JSON;
-
-statement ok
-DROP TABLE mytable;
-
-
 # `DEBEZIUM_MONGO_JSON` requires the source table have `_id` and `payload` columns.
 statement error
 create source s (
@@ -236,7 +207,6 @@ create source s (
   connector = 'kafka',
   topic = 'kafka_1_partition_topic',
   properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
 ) FORMAT DEBEZIUM_MONGO ENCODE JSON;
 
 # `DEBEZIUM_MONGO_JSON` requires the `_id` column is primary key.
@@ -248,7 +218,6 @@ create source s (
   connector = 'kafka',
   topic = 'kafka_1_partition_topic',
   properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
 ) FORMAT DEBEZIUM_MONGO ENCODE JSON;
 
 # `DEBEZIUM_MONGO_JSON` requires the `payload` column is jsonb type.
@@ -260,25 +229,4 @@ create source s (
   connector = 'kafka',
   topic = 'kafka_1_partition_topic',
   properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
 ) FORMAT DEBEZIUM_MONGO ENCODE JSON;
-
-statement ok
-create source s (
-  column1 varchar
-) with (
-  connector = 'kafka',
-  topic = 'kafka_1_partition_topic',
-  properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'mock',
-) FORMAT PLAIN ENCODE JSON;
-
-# Drop a connection in use
-statement error
-drop connection mock;
-
-statement ok
-drop source s;
-
-statement ok
-drop connection mock;
diff --git a/e2e_test/source_legacy/basic/kafka.slt b/e2e_test/source_legacy/basic/kafka.slt
index 227c0aa46bac1..4e597a98801df 100644
--- a/e2e_test/source_legacy/basic/kafka.slt
+++ b/e2e_test/source_legacy/basic/kafka.slt
@@ -1,3 +1,6 @@
+statement ok
+SET streaming_use_shared_source TO false;
+
 # We don't support CSV header for Kafka
 statement error CSV HEADER is not supported when creating table with Kafka connector
 create table s0 (v1 int, v2 varchar) with (
@@ -496,6 +499,16 @@ WITH (
     scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE JSON
 
+statement ok
+CREATE TABLE test_include_payload_only
+INCLUDE payload
+WITH (
+    connector = 'kafka',
+    topic = 'kafka_1_partition_topic',
+    properties.bootstrap.server = 'message_queue:29092',
+    scan.startup.mode = 'earliest'
+) FORMAT PLAIN ENCODE JSON
+
 statement ok
 flush;
 
@@ -908,5 +921,11 @@ drop table source_with_rdkafka_props;
 statement ok
 drop table debezium_ignore_key;
 
+statement ok
+drop table test_include_payload_only;
+
 statement ok
 drop table test_include_payload;
+
+statement ok
+SET streaming_use_shared_source TO true;
diff --git a/e2e_test/source_legacy/basic/old_row_format_syntax/ddl.slt b/e2e_test/source_legacy/basic/old_row_format_syntax/ddl.slt
index 0fe67a8504b5d..79c3553c38c70 100644
--- a/e2e_test/source_legacy/basic/old_row_format_syntax/ddl.slt
+++ b/e2e_test/source_legacy/basic/old_row_format_syntax/ddl.slt
@@ -152,35 +152,6 @@ s
 statement ok
 drop table s
 
-# Test create source with connection
-statement ok
-CREATE CONNECTION mock WITH (type = 'privatelink', provider = 'mock');
-
-# Reference to non-existant connection
-statement error
-create source s (
-  column1 varchar
-) with (
-  connector = 'kafka',
-  topic = 'kafka_1_partition_topic',
-  properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
-) ROW FORMAT JSON;
-
-statement ok
-CREATE TABLE mytable (
-  column1 varchar
-) with (
-  connector = 'kafka',
-  topic = 'kafka_1_partition_topic',
-  properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'mock'
-) ROW FORMAT JSON;
-
-statement ok
-DROP TABLE mytable;
-
-
 # `DEBEZIUM_MONGO_JSON` requires the source table have `_id` and `payload` columns.
 statement error
 create source s (
@@ -190,7 +161,6 @@ create source s (
   connector = 'kafka',
   topic = 'kafka_1_partition_topic',
   properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
 ) ROW FORMAT DEBEZIUM_MONGO_JSON;
 
 # `DEBEZIUM_MONGO_JSON` requires the `_id` column is primary key.
@@ -202,7 +172,6 @@ create source s (
   connector = 'kafka',
   topic = 'kafka_1_partition_topic',
   properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
 ) ROW FORMAT DEBEZIUM_MONGO_JSON;
 
 # `DEBEZIUM_MONGO_JSON` requires the `payload` column is jsonb type.
@@ -214,7 +183,6 @@ create source s (
   connector = 'kafka',
   topic = 'kafka_1_partition_topic',
   properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'nonexist',
 ) ROW FORMAT DEBEZIUM_MONGO_JSON;
 
 statement ok
@@ -224,15 +192,7 @@ create source s (
   connector = 'kafka',
   topic = 'kafka_1_partition_topic',
   properties.bootstrap.server = 'message_queue:29092',
-  connection.name = 'mock',
 ) ROW FORMAT JSON;
 
-# Drop a connection in use
-statement error
-drop connection mock;
-
 statement ok
 drop source s;
-
-statement ok
-drop connection mock;
diff --git a/e2e_test/source_legacy/cdc/cdc.share_stream.slt b/e2e_test/source_legacy/cdc/cdc.share_stream.slt
index a2de0a5e8168d..cf1000957b6fb 100644
--- a/e2e_test/source_legacy/cdc/cdc.share_stream.slt
+++ b/e2e_test/source_legacy/cdc/cdc.share_stream.slt
@@ -4,9 +4,17 @@ control substitution on
 system ok
 mysql --protocol=tcp -u root -e "DROP DATABASE IF EXISTS mytest; CREATE DATABASE mytest;"
 
+system ok
+mysql --protocol=tcp -u root -e "DROP DATABASE IF EXISTS kdb; CREATE DATABASE kdb;"
+
 system ok
 mysql --protocol=tcp -u root mytest < e2e_test/source_legacy/cdc/mysql_create.sql
 
+system ok
+mysql --protocol=tcp -u root kdb -e " CREATE TABLE kt1 (id int primary key, v1 varchar(32));
+                                      INSERT INTO kt1 VALUES (1,'aaa'),(2,'bbb');
+"
+
 # generate data to mysql
 system ok
 mysql --protocol=tcp -u root mytest < e2e_test/source_legacy/cdc/mysql_init_data.sql
@@ -28,6 +36,7 @@ create source mysql_mytest with (
  server.id = '5601'
 );
 
+
 statement error Should not create MATERIALIZED VIEW or SELECT directly on shared CDC source
 create materialized view mv as select * from mysql_mytest;
 
@@ -70,6 +79,19 @@ SINGLE {STREAM_SCAN}
 SINGLE {CDC_FILTER}
 HASH {SOURCE,DML}
 
+
+statement ok
+create source mysql_source with (
+ connector = 'mysql-cdc',
+ hostname = '${MYSQL_HOST:localhost}',
+ port = '${MYSQL_TCP_PORT:8306}',
+ username = 'rwcdc',
+ password = secret mysql_pwd,
+ database.name = 'mytest,kdb',
+ server.id = '5602'
+);
+
+
 statement ok
 CREATE TABLE IF NOT EXISTS mysql_all_types(
     c_boolean boolean,
@@ -112,6 +134,10 @@ create table orders_test (
    PRIMARY KEY (order_id)
 ) from mysql_mytest table 'mytest.orders';
 
+
+statement ok
+create table kt1 (*) from mysql_source table 'kdb.kt1';
+
 statement ok
 create materialized view products_test_cnt as select count(*) as cnt from rw.products_test;
 
@@ -121,6 +147,9 @@ create materialized view orders_test_cnt as select count(*) as cnt from orders_t
 system ok
 mysql --protocol=tcp -u root mytest -e "INSERT INTO products VALUES(default, 'Juice', '100ml Juice');"
 
+system ok
+mysql --protocol=tcp -u root kdb -e "INSERT INTO kt1 VALUES (3, 'ccc'),(4, 'ddd');"
+
 system ok
 mysql --protocol=tcp -u root mytest -e "FLUSH LOGS"
 
@@ -146,6 +175,16 @@ select count(*) from orders_no_backfill
 ----
 0
 
+
+query IT
+select * from kt1 order by id;
+----
+1 aaa
+2 bbb
+3 ccc
+4 ddd
+
+
 # check ingestion results
 query I
 SELECT * from products_test_cnt
@@ -193,10 +232,10 @@ SELECT order_id,order_date,customer_name,product_id FROM orders_test order by or
 10003 2020-07-30 12:00:30 Edward 106
 
 query IIIIITTTTTTTTT
-SELECT c_tinyint, c_smallint, c_mediumint, c_integer, c_bigint, c_decimal, c_float, c_double, c_char_255, c_varchar_10000, c_date, c_time, c_datetime, c_timestamp FROM mysql_all_types order by c_bigint;
+SELECT c_boolean, c_bit, c_tinyint, c_smallint, c_mediumint, c_integer, c_bigint, c_decimal, c_float, c_double, c_char_255, c_varchar_10000, c_date, c_time, c_datetime, c_timestamp FROM mysql_all_types order by c_bigint;
 ----
--128 -32767 -8388608 -2147483647 -9223372036854775807 -10 -10000 -10000 a b 1001-01-01 00:00:00 1998-01-01 00:00:00 1970-01-01 00:00:01+00:00
-NULL NULL -8388608 -2147483647 9223372036854775806 -10 -10000 -10000 c d 1001-01-01 NULL 2000-01-01 00:00:00 NULL
+t t -128 -32767 -8388608 -2147483647 -9223372036854775807 -10 -10000 -10000 a b 1001-01-01 00:00:00 1998-01-01 00:00:00 1970-01-01 00:00:01+00:00
+f f NULL NULL -8388608 -2147483647 9223372036854775806 -10 -10000 -10000 c d 1001-01-01 NULL 2000-01-01 00:00:00 NULL
 
 statement ok
 create secret pg_pwd with (
diff --git a/e2e_test/source_legacy/cdc_inline/auto_schema_map_mysql.slt b/e2e_test/source_legacy/cdc_inline/auto_schema_map_mysql.slt
index 08afa5d1988a7..e353b24996ed4 100644
--- a/e2e_test/source_legacy/cdc_inline/auto_schema_map_mysql.slt
+++ b/e2e_test/source_legacy/cdc_inline/auto_schema_map_mysql.slt
@@ -32,7 +32,7 @@ mysql --protocol=tcp -u root mytest -e "
         c_time time,
         c_datetime datetime,
         c_timestamp timestamp,
-        c_enum ENUM('happy','sad','ok'),
+        c_enum ENUM('happy','sad','ok') DEFAULT 'ok',
         c_json JSON,
         PRIMARY KEY (c_boolean,c_Bigint,c_date)
     );
@@ -174,8 +174,8 @@ SELECT
     c_binary_255
 FROM rw_mysql_types_test order by c_boolean;
 ----
-0 NULL NULL NULL -8388608 -2147483647 9223372036854775806 -10 -10000 -10000 c d \x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
-1 NULL -128 -32767 -8388608 -2147483647 -9223372036854775807 -10 -10000 -10000 a b \x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+0 f NULL NULL -8388608 -2147483647 9223372036854775806 -10 -10000 -10000 c d \x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
+1 t -128 -32767 -8388608 -2147483647 -9223372036854775807 -10 -10000 -10000 a b \x000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
 
 query TTTTTTTT
 SELECT
diff --git a/e2e_test/streaming/changelog.slt b/e2e_test/streaming/changelog.slt
index 649478ffcb981..9eca1a7ae5d4a 100644
--- a/e2e_test/streaming/changelog.slt
+++ b/e2e_test/streaming/changelog.slt
@@ -10,8 +10,9 @@ create table t2 (v1 int, v2 int);
 statement ok
 create table t3 (v1 int primary key, v2 int);
 
+# test public.t1 due to https://github.com/risingwavelabs/risingwave/issues/18747
 statement ok
-create materialized view mv1 as with sub as changelog from t1 select * from sub;
+create materialized view mv1 as with sub as changelog from public.t1 select * from sub;
 
 statement ok
 create materialized view mv2 as with sub as changelog from t2 select * from sub;
diff --git a/e2e_test/streaming/temporal_join/append_only/nested_loop.slt b/e2e_test/streaming/temporal_join/append_only/nested_loop.slt
new file mode 100644
index 0000000000000..6c9e28799f304
--- /dev/null
+++ b/e2e_test/streaming/temporal_join/append_only/nested_loop.slt
@@ -0,0 +1,96 @@
+statement ok
+SET RW_IMPLICIT_FLUSH TO true;
+
+statement ok
+create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+
+statement ok
+create table version(id2 int, a2 int, b2 int, primary key (id2));
+
+statement ok
+create materialized view v1 as select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 > a2;
+
+statement ok
+create materialized view v2 as select id1, a1, id2, a2 from stream cross join version FOR SYSTEM_TIME AS OF PROCTIME();
+
+statement ok
+insert into stream values(1, 14, 111);
+
+statement ok
+insert into version values(1, 12, 122);
+
+statement ok
+insert into stream values(2, 13, 133);
+
+statement ok
+delete from version;
+
+query IIII rowsort
+select * from v1;
+----
+2 13 1 12
+
+query IIII rowsort
+select * from v2;
+----
+2 13 1 12
+
+statement ok
+insert into version values(2, 10, 102);
+
+statement ok
+insert into stream values(3, 9, 222);
+
+query IIII rowsort
+select * from v1;
+----
+2 13 1 12
+
+query IIII rowsort
+select * from v2;
+----
+2 13 1 12
+3 9 2 10
+
+statement ok
+drop materialized view v1;
+
+statement ok
+drop materialized view v2;
+
+statement ok
+delete from version where id2 = 2;
+
+statement ok
+insert into version values(4, 10, 104);
+
+statement ok
+create materialized view v1 as select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 > a2
+
+statement ok
+create materialized view v2 as select id1, a1, id2, a2 from stream cross join version FOR SYSTEM_TIME AS OF PROCTIME();
+
+query IIII rowsort
+select * from v1;
+----
+1 14 4 10
+2 13 4 10
+
+query IIII rowsort
+select * from v2;
+----
+1 14 4 10
+2 13 4 10
+3 9 4 10
+
+statement ok
+drop materialized view v1;
+
+statement ok
+drop materialized view v2;
+
+statement ok
+drop table stream;
+
+statement ok
+drop table version;
diff --git a/e2e_test/subscription/create_table_and_subscription.slt b/e2e_test/subscription/create_table_and_subscription.slt
index 8ba73fae7ec91..02b13f1deaf9b 100644
--- a/e2e_test/subscription/create_table_and_subscription.slt
+++ b/e2e_test/subscription/create_table_and_subscription.slt
@@ -20,4 +20,19 @@ statement ok
 create subscription sub2 from t3 with(retention = '1D');
 
 statement ok
-create sink s1 into t3 from t2;
\ No newline at end of file
+create sink s1 into t3 from t2;
+
+statement ok
+create table t4 (v1 int, v2 int, v3 int, v4 int);
+
+statement ok
+create materialized view mv4 as select v4,v2 from t4;
+
+statement ok
+create subscription sub4 from mv4 with(retention = '1D');
+
+statement ok
+create table t5 (v1 int, v2 int, v3 int, v4 int, primary key (v1, v2));
+
+statement ok
+create subscription sub5 from t5 with(retention = '1D');
\ No newline at end of file
diff --git a/e2e_test/subscription/drop_table_and_subscription.slt b/e2e_test/subscription/drop_table_and_subscription.slt
index d89401c9cddc8..f75b58accc961 100644
--- a/e2e_test/subscription/drop_table_and_subscription.slt
+++ b/e2e_test/subscription/drop_table_and_subscription.slt
@@ -14,4 +14,20 @@ statement ok
 drop table t3;
 
 statement ok
-drop table t2;
\ No newline at end of file
+drop table t2;
+
+
+statement ok
+drop subscription sub4;
+
+statement ok
+drop materialized view mv4;
+
+statement ok
+drop table t4;
+
+statement ok
+drop subscription sub5;
+
+statement ok
+drop table t5;
\ No newline at end of file
diff --git a/e2e_test/subscription/main.py b/e2e_test/subscription/main.py
index a3079e2e6847e..e8549ae85025b 100644
--- a/e2e_test/subscription/main.py
+++ b/e2e_test/subscription/main.py
@@ -365,7 +365,6 @@ def test_block_cursor():
         user="root",
         database="dev"
     )
-
     execute_insert("declare cur subscription cursor for sub2 full",conn)
     execute_insert("insert into t2 values(1,1)",conn)
     execute_insert("flush",conn)
@@ -402,6 +401,148 @@ def insert_into_table():
     )
     execute_insert("insert into t2 values(10,10)",conn)
 
+def test_order_table_with_pk():
+    print(f"test_order_table_with_pk")
+    create_table_subscription()
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+    execute_insert("insert into t2 values(6,6),(3,3),(5,5),(4,4),(7,7)",conn)
+    execute_insert("flush",conn)
+    execute_insert("declare cur subscription cursor for sub2 full",conn)
+    row = execute_query("fetch 5 from cur",conn)
+    assert len(row) == 5
+    check_rows_data([3,3],row[0],"Insert")
+    check_rows_data([4,4],row[1],"Insert")
+    check_rows_data([5,5],row[2],"Insert")
+    check_rows_data([6,6],row[3],"Insert")
+    check_rows_data([7,7],row[4],"Insert")
+    execute_insert("insert into t2 values(16,16),(13,13),(15,15),(14,14),(17,17)",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch 5 from cur",conn)
+    assert len(row) == 5
+    check_rows_data([13,13],row[0],"Insert")
+    check_rows_data([14,14],row[1],"Insert")
+    check_rows_data([15,15],row[2],"Insert")
+    check_rows_data([16,16],row[3],"Insert")
+    check_rows_data([17,17],row[4],"Insert")
+    execute_insert("update t2 set v2 = 100 where v1 > 10",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch 10 from cur",conn)
+    assert len(row) == 10
+    check_rows_data([13,13],row[0],"UpdateDelete")
+    check_rows_data([13,100],row[1],"UpdateInsert")
+    check_rows_data([14,14],row[2],"UpdateDelete")
+    check_rows_data([14,100],row[3],"UpdateInsert")
+    check_rows_data([15,15],row[4],"UpdateDelete")
+    check_rows_data([15,100],row[5],"UpdateInsert")
+    check_rows_data([16,16],row[6],"UpdateDelete")
+    check_rows_data([16,100],row[7],"UpdateInsert")
+    check_rows_data([17,17],row[8],"UpdateDelete")
+    check_rows_data([17,100],row[9],"UpdateInsert")
+    drop_table_subscription()
+
+def test_order_table_with_row_id():
+    print(f"test_order_table_with_pk")
+    create_table_subscription()
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+    execute_insert("insert into t1 values(6,6),(3,3),(5,5),(4,4),(7,7)",conn)
+    execute_insert("flush",conn)
+    execute_insert("declare cur subscription cursor for sub full",conn)
+    row = execute_query("fetch 10 from cur",conn)
+    ex_row = execute_query("select v1, v2 from t1 order by _row_id",conn)
+    assert len(row) == 6
+    assert len(ex_row) == 6
+    check_rows_data(ex_row[0],row[0],"Insert")
+    check_rows_data(ex_row[1],row[1],"Insert")
+    check_rows_data(ex_row[2],row[2],"Insert")
+    check_rows_data(ex_row[3],row[3],"Insert")
+    check_rows_data(ex_row[4],row[4],"Insert")
+    execute_insert("insert into t1 values(16,16),(13,13),(15,15),(14,14),(17,17)",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch 5 from cur",conn)
+    ex_row = execute_query("select v1, v2 from t1 where v1 > 10 order by _row_id",conn)
+    assert len(row) == 5
+    assert len(ex_row) == 5
+    check_rows_data(ex_row[0],row[0],"Insert")
+    check_rows_data(ex_row[1],row[1],"Insert")
+    check_rows_data(ex_row[2],row[2],"Insert")
+    check_rows_data(ex_row[3],row[3],"Insert")
+    check_rows_data(ex_row[4],row[4],"Insert")
+    drop_table_subscription()
+
+def test_order_mv():
+    print(f"test_order_mv")
+    create_table_subscription()
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+    execute_insert("insert into t4 values(6,6,6,6),(3,3,3,3),(5,5,5,5),(4,4,4,4),(7,7,7,7)",conn)
+    execute_insert("flush",conn)
+    execute_insert("declare cur subscription cursor for sub4 full",conn)
+    row = execute_query("fetch 5 from cur",conn)
+    ex_row = execute_query("select v4, v2 from t4 order by _row_id",conn)
+    assert len(row) == 5
+    assert len(ex_row) == 5
+    check_rows_data(ex_row[0],row[0],"Insert")
+    check_rows_data(ex_row[1],row[1],"Insert")
+    check_rows_data(ex_row[2],row[2],"Insert")
+    check_rows_data(ex_row[3],row[3],"Insert")
+    check_rows_data(ex_row[4],row[4],"Insert")
+    execute_insert("insert into t4 values(16,16,16,16),(13,13,13,13),(15,15,15,15),(14,14,14,14),(17,17,17,17)",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch 5 from cur",conn)
+    ex_row = execute_query("select v4, v2 from t4 where v2 > 10 order by _row_id",conn)
+    assert len(row) == 5
+    assert len(ex_row) == 5
+    check_rows_data(ex_row[0],row[0],"Insert")
+    check_rows_data(ex_row[1],row[1],"Insert")
+    check_rows_data(ex_row[2],row[2],"Insert")
+    check_rows_data(ex_row[3],row[3],"Insert")
+    check_rows_data(ex_row[4],row[4],"Insert")
+    drop_table_subscription()
+
+def test_order_multi_pk():
+    print(f"test_order_mutil_pk")
+    create_table_subscription()
+    conn = psycopg2.connect(
+        host="localhost",
+        port="4566",
+        user="root",
+        database="dev"
+    )
+    execute_insert("insert into t5 values(6,6,6,6),(6,3,3,3),(5,5,5,5),(5,4,4,4),(7,7,7,7)",conn)
+    execute_insert("flush",conn)
+    execute_insert("declare cur subscription cursor for sub5 full",conn)
+    row = execute_query("fetch 5 from cur",conn)
+    assert len(row) == 5
+    check_rows_data([5,4,4,4],row[0],"Insert")
+    check_rows_data([5,5,5,5],row[1],"Insert")
+    check_rows_data([6,3,3,3],row[2],"Insert")
+    check_rows_data([6,6,6,6],row[3],"Insert")
+    check_rows_data([7,7,7,7],row[4],"Insert")
+    execute_insert("insert into t5 values(16,16,16,16),(16,13,13,13),(15,15,15,15),(15,14,14,14),(17,17,17,17)",conn)
+    execute_insert("flush",conn)
+    row = execute_query("fetch 5 from cur",conn)
+    assert len(row) == 5
+    check_rows_data([15,14,14,14],row[0],"Insert")
+    check_rows_data([15,15,15,15],row[1],"Insert")
+    check_rows_data([16,13,13,13],row[2],"Insert")
+    check_rows_data([16,16,16,16],row[3],"Insert")
+    check_rows_data([17,17,17,17],row[4],"Insert")
+    drop_table_subscription()
+
 if __name__ == "__main__":
     test_cursor_snapshot()
     test_cursor_op()
@@ -413,4 +554,8 @@ def insert_into_table():
     test_cursor_with_table_alter()
     test_cursor_fetch_n()
     test_rebuild_table()
+    test_order_table_with_pk()
+    test_order_table_with_row_id()
+    test_order_mv()
+    test_order_multi_pk()
     test_block_cursor()
diff --git a/e2e_test/time_travel/basic.slt b/e2e_test/time_travel/basic.slt
index 962fd8e096a59..6dd65cf7125d3 100644
--- a/e2e_test/time_travel/basic.slt
+++ b/e2e_test/time_travel/basic.slt
@@ -1,6 +1,9 @@
 statement ok
 SET RW_IMPLICIT_FLUSH TO true;
 
+statement error time_travel_retention_ms cannot be less than 600000
+ALTER SYSTEM SET time_travel_retention_ms to 10;
+
 statement ok
 CREATE TABLE t (k INT);
 
diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py
index 057314785bd6c..ef6d5abf51119 100644
--- a/grafana/risingwave-dev-dashboard.dashboard.py
+++ b/grafana/risingwave-dev-dashboard.dashboard.py
@@ -594,6 +594,65 @@ def section_compaction(outer_panels):
                         ),
                     ],
                 ),
+
+                panels.timeseries_count(
+                    "Compaction Group Count",
+                    "The number of compaction groups",
+                    [
+                        panels.target(
+                            f"{metric('storage_compaction_group_count')}",
+                            "compaction group count",
+                        ),
+                    ],
+                ),
+
+                panels.timeseries_bytes(
+                    "Compaction Group Size",
+                    "The size of compaction group",
+                    [
+                        panels.target(
+                            f"sum({metric('storage_compaction_group_size')}) by (group)",
+                            "compaction group size - cg{{group}}",
+                        ),
+                    ],
+                ),
+
+                panels.timeseries_count(
+                    "Compaction Group File Count",
+                    "The file count of compaction group",
+                    [
+                        panels.target(
+                            f"sum({metric('storage_compaction_group_file_count')}) by (group)",
+                            "compaction group file count - cg{{group}}",
+                        ),
+                    ],
+                ),
+
+                panels.timeseries_bytes(
+                    "Compaction Group Throughput",
+                    "The throughput of compaction group",
+                    [
+                        panels.target(
+                            f"sum({metric('storage_compaction_group_throughput')}) by (group)",
+                            "compaction group throughput - cg{{group}}",
+                        ),
+                    ],
+                ),
+
+                 panels.timeseries_count(
+                    "Compaction Group Schedule",
+                    "The times of move_state_table occurs",
+                    [
+                        panels.target(
+                            f"sum({table_metric('storage_split_compaction_group_count')}) by (group)",
+                            "split compaction group - left cg{{group}}",
+                        ),
+                        panels.target(
+                            f"sum({table_metric('storage_merge_compaction_group_count')}) by (group)",
+                            "merge compaction group - left cg{{group}}",
+                        ),
+                    ],
+                ),
             ],
         )
     ]
@@ -933,7 +992,7 @@ def section_streaming(outer_panels):
                 ),
                 panels.timeseries_rowsps(
                     "Backfill Snapshot Read Throughput(rows)",
-                    "Total number of rows that have been read from the backfill snapshot",
+                    "Rows/sec that we read from the backfill snapshot",
                     [
                         panels.target(
                             f"rate({table_metric('stream_backfill_snapshot_read_row_count')}[$__rate_interval])",
@@ -947,6 +1006,23 @@ def section_streaming(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_rowsps(
+                    "Backfill Snapshot Read Throughput(rows) by MV",
+                    "Rows/sec that we read from the backfill snapshot by materialized view",
+                    [
+                        panels.target(
+                            f"""
+                                sum by (table_id) (
+                                  rate({metric('stream_backfill_snapshot_read_row_count', node_filter_enabled=False, table_id_filter_enabled=True)}[$__rate_interval])
+                                )
+                                * on(table_id) group_left(table_name) (
+                                  group({metric('table_info', node_filter_enabled=False)}) by (table_name, table_id)
+                                )
+                            """,
+                            "table_name={{table_name}} table_id={{table_id}}",
+                        )
+                    ],
+                ),
                 panels.timeseries_rowsps(
                     "Backfill Upstream Throughput(rows)",
                     "Total number of rows that have been output from the backfill upstream",
@@ -1321,16 +1397,20 @@ def section_streaming_actors(outer_panels: Panels):
                     ],
                 ),
                 panels.timeseries_actor_ops(
-                    "Over Window Executor Compute Count",
+                    "Over Window Executor State Computation",
                     "",
                     [
+                        panels.target(
+                            f"sum(rate({table_metric('stream_over_window_accessed_entry_count')}[$__rate_interval])) by (table_id, fragment_id)",
+                            "accessed entry count - table {{table_id}} fragment {{fragment_id}}",
+                        ),
                         panels.target(
                             f"sum(rate({table_metric('stream_over_window_compute_count')}[$__rate_interval])) by (table_id, fragment_id)",
                             "compute count - table {{table_id}} fragment {{fragment_id}}",
                         ),
                         panels.target(
-                            f"sum(rate({table_metric('stream_over_window_same_result_count')}[$__rate_interval])) by (table_id, fragment_id)",
-                            "same result count - table {{table_id}} fragment {{fragment_id}}",
+                            f"sum(rate({table_metric('stream_over_window_same_output_count')}[$__rate_interval])) by (table_id, fragment_id)",
+                            "same output count - table {{table_id}} fragment {{fragment_id}}",
                         ),
                     ],
                 ),
@@ -3383,20 +3463,6 @@ def section_hummock_manager(outer_panels):
                         ),
                     ],
                 ),
-                panels.timeseries_count(
-                    "Compaction Group Schedule",
-                    "The times of move_state_table occurs",
-                    [
-                        panels.target(
-                            f"sum({table_metric('storage_split_compaction_group_count')}) by (group)",
-                            "split compaction group cg{{group}}",
-                        ),
-                        panels.target(
-                            f"sum({table_metric('storage_merge_compaction_group_count')}) by (group)",
-                            "merge compaction group cg{{group}}",
-                        ),
-                    ],
-                ),
                 panels.timeseries_count(
                     "State Table Count",
                     "The number of state_tables in each CG",
diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json
index 8efff55d606a3..2315a73da65e9 100644
--- a/grafana/risingwave-dev-dashboard.json
+++ b/grafana/risingwave-dev-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","query":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_inflight_barrier_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_in_flight_barrier {{table_id}}","metric":"","query":"meta_snapshot_backfill_inflight_barrier_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p50 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p90 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p99 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p999 table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_pmax table_id[{{table_id}}] {{barrier_type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","query":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","query":"rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_snapshot_backfill_consume_snapshot_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} {{stage}} @ {{instance}}","metric":"","query":"rate(stream_snapshot_backfill_consume_snapshot_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p50 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p90 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p99 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p999 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_pmax table_id[{{table_id}}]","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p50 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p90 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p99 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p999 table_id[{{table_id}}]","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_pmax table_id[{{table_id}}]","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_upstream_lag{job=~\"$job\",instance=~\"$node\"} / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag @ {{table_id}}","metric":"","query":"meta_snapshot_backfill_upstream_lag{job=~\"$job\",instance=~\"$node\"} / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Snapshot Backfill Lag","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":40,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed auto schema change attempts of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","query":"sum(auto_schema_change_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of succeeded auto schema change of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_success_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","query":"sum(auto_schema_change_success_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of Auto Schema Change Process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50{{table_id}} - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99{{table_id}} - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax{{table_id}} - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Latency (sec)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":48,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id)                             / ignoring (downstream_fragment_id) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)                             / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id)                             / ignoring (downstream_fragment_id) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)                             / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table total {{table_id}}: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{actor_id!='',job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} table {{table_id}}: {{desc}}","metric":"","query":"stream_memory_usage{actor_id!='',job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_compute_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_compute_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_same_result_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"same result count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_same_result_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Compute Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_state_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_state_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg state cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_state_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_state_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id, wait_side, executor)                             / ignoring (wait_side, executor) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment avg {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"sum(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id, wait_side, executor)                             / ignoring (wait_side, executor) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_align_duration_ns{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","query":"rate(stream_barrier_align_duration_ns{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","query":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment total {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":75,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":92,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":99,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":106,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","query":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","query":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","query":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_hit{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_safe_version_hit{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_miss{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_safe_version_miss{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Safe Version Fetch Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":129,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":144,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing bloom_filter size, sstable file size, sstable block size etc.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Bloom Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable file size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable File Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable block size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Block Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing avg key and value count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Avg Key And Value Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":171,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":180,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","query":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","query":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","query":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","query":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":208,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_time_travel_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by time travel","metric":"","query":"storage_time_travel_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_split_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"split compaction group cg{{group}}","metric":"","query":"sum(storage_split_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_merge_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge compaction group cg{{group}}","metric":"","query":"sum(storage_merge_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Schedule","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The latency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_time_travel_version_replay_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_avg","metric":"","query":"rate(storage_time_travel_version_replay_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The frequency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_ops","metric":"","query":"sum(rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":229,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":232,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":236,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":239,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":243,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":247,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":255,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of valid and invalid subscription cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"invalid_subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"invalid_subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subsription Cursor Nums","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The subscription error num of cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subscription_cursor_error_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"subscription_cursor_error_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Error Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The amount of time a query exists inside the cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Query Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of declare","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Declare Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Since the last fetch, the time up to now","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Last Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":270,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":281,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rows sent by remote sink to the Java connector process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Remote Sink (Java) Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store unconsumed min epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}})","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} @ {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, sink_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, sink_id, actor_id, sink_name)","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, sink_id, actor_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","query":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":299,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":318,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":323,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":326,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}}","metric":"","query":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":330,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":331,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":332,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"}) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write @ {{sink_name}}","metric":"","query":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"}) by (sink_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total write","metric":"","query":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":333,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_read_bytes{job=~\"$job\",instance=~\"$node\"}) by (table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read @ {{table_name}}","metric":"","query":"sum(iceberg_read_bytes{job=~\"$job\",instance=~\"$node\"}) by (table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(nimtable_read_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total read","metric":"","query":"sum(nimtable_read_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Read Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":334,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":335,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","query":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":336,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":337,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":338,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":339,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":340,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","query":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(avg by(namespace, pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\"})) / (  sum by(namespace, pod) (kube_pod_container_resource_limits{namespace=~\"$namespace\", pod=~\"$pod\", container=\"$component\", resource=\"memory\", unit=\"byte\"}))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg memory usage @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"CPU usage relative to k8s resource limit of container. Only works in K8s environment","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(container_cpu_usage_seconds_total{namespace=~\"$namespace\",container=~\"$component\",pod=~\"$pod\"}[$__rate_interval])) by (namespace, pod)) / (sum(kube_pod_container_resource_limits{namespace=~\"$namespace\",pod=~\"$pod\",container=~\"$component\", resource=\"cpu\"}) by (namespace, pod))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU relative","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":12,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_inflight_barrier_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_in_flight_barrier {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p50 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p90 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p99 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_p999 table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, barrier_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_latency_pmax table_id[{{table_id}}] {{barrier_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration from the last committed barrier's epoch time to the current time. This metric reflects the data freshness of the system. During this time, no new data has been committed.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"timestamp(last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}) - last_committed_barrier_time{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_pending_time","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier pending time (secs)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragment_id={{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min(source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"} - on(source_id, partition) group_right() source_latest_message_id{job=~\"$job\",instance=~\"$node\"}, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second. For sinks with 'sink_decouple = true', please refer to the 'Sink Metrics' section","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition *","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_mview_input_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - actor {{actor_id}} fragment_id {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Rows/sec that we read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_snapshot_backfill_consume_snapshot_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} {{stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Rows/sec that we read from the backfill snapshot by materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"\n                                sum by (table_id) (\n                                  rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\"}[$__rate_interval])\n                                )\n                                * on(table_id) group_left(table_name) (\n                                  group(table_info) by (table_name, table_id)\n                                )\n                            ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_name={{table_name}} table_id={{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows) by MV","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p50 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p90 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p99 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_p999 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_barrier_wait_commit_latency_pmax table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p50 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p90 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p99 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_p999 table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_snapshot_backfill_upstream_wait_progress_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"snapshot_backfill_upstream_wait_progress_latency_pmax table_id[{{table_id}}]","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"meta_snapshot_backfill_upstream_lag{job=~\"$job\",instance=~\"$node\"} / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag @ {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Snapshot Backfill Lag","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":41,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed auto schema change attempts of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of succeeded auto schema change of CDC Table","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(auto_schema_change_success_cnt{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of Auto Schema Change Process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(auto_schema_change_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax{{table_id}} - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Auto Schema Change Latency (sec)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":49,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id)                             / ignoring (downstream_fragment_id) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)                             / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment total {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table total {{table_id}}: {{desc}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{actor_id!='',job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} table {{table_id}}: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{actor_id!='',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - actor {{actor_id}} table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_accessed_entry_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"accessed entry count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_compute_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_same_output_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"same output count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor State Computation","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_state_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_state_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg state cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_barrier_align_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id, wait_side, executor)                             / ignoring (wait_side, executor) group_left sum(stream_actor_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment avg {{fragment_id}} {{wait_side}} {{executor}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_align_duration_ns{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} fragment {{fragment_id}} {{wait_side}} {{executor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Barrier Align Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_merge_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, job)(rate(stream_merge_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,job) (rate(stream_merge_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Merger Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss  - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups  - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment total {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{actor_id!='',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":76,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":93,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during computation. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during source data ingestion. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors that happened during data sink out. Check the logs for detailed error message.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Errors by Type","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":100,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"compute_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Disk throughputs of spilling-out in the bacth query engine","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_spill_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Spill Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":107,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_prefetch_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"prefetch cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss ratio - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the estimated hit ratio of a block while in the block cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.1, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p10 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.25, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p25 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.5, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.75, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p75 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(0.9, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_max(histogram_quantile(1.0, sum(rate(block_efficiency_histogram_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le,job,instance)), 1)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache efficienfy - p100 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Cache Efficiency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, iter_type) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{iter_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count  - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id, iter_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{iter_type}} - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id, iter_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{iter_type}} {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_in_progress_counts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Existing {{iter_type}} count @ {{table_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_log_op_type_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, op_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter_log op count @ {{table_id}} {{op_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_hit{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_safe_version_miss{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Safe Version Fetch Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploader imm size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unflushed imm size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance) - sum(state_store_uploader_imm_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"orphan imm size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_old_value_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"old value size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg Sync duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_upload_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax upload task duration - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_syncing_epoch_count{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"syncing epoch count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_event_handler_pending_event{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handler pending event number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_event_handler_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, event_type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax {{event_type}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_uploader_wait_poll_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax finished_task_wait_poll {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Event handle latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":145,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The  of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing bloom_filter size, sstable file size, sstable block size etc.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_bloom_filter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom_filter_size_p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Bloom Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable file size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_file_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_file_size_p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable File Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing sstable block size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(rate(compactor_sstable_block_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sstable_block_size_p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Block Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"For observing avg key and value count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))  / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Avg Key And Value Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compaction groups","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_compaction_group_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of compaction group","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":104},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compaction_group_size{job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group size - cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The file count of compaction group","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":112},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compaction_group_file_count{job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group file count - cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The throughput of compaction group","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":112},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compaction_group_throughput{job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction group throughput - cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":120},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_split_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"split compaction group - left cg{{group}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_merge_compaction_group_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge compaction group - left cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Group Schedule","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":177,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(read|streaming_read_init|metadata)',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'(upload|streaming_upload|streaming_upload_init|streaming_upload_finish|delete_objects|list|delete)',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":186,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_hybrid_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - hybrid - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_hybrid_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_hybrid_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - hybrid - hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hybrid Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_memory_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_memory_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - memory - hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_inner_op_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - storage - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) / (sum(rate(foyer_storage_op_total{op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance) + sum(rate(foyer_storage_op_total{op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - storage - hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_region{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance) * on(name, instance) group_left() avg(foyer_storage_region_size_bytes{job=~\"$job\",instance=~\"$node\"}) by (name, type, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - {{type}} region - size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Region Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_disk_io_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, name, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_disk_io_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (name, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{name}} - disk - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Disk Op Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) >= 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Item numbers of the recent filter.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":96},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recent_filter_items{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"items @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":104},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recent_filter_ops{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recent filter {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recent Filter Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":214,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{method}} @ {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_time_travel_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by time travel","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The latency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_time_travel_version_replay_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_time_travel_version_replay_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The frequency of replaying a hummock version for time travel","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_time_travel_version_replay_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"time_travel_version_replay_ops","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Time Travel Replay Ops","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":234,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":237,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":241,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":244,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":248,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":252,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":260,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of valid and invalid subscription cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"invalid_subsription_cursor_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subsription Cursor Nums","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The subscription error num of cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"subscription_cursor_error_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Error Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The amount of time a query exists inside the cursor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_query_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Query Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of declare","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_declare_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Declare Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Subscription cursor duration of fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Since the last fetch, the time up to now","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{subscription_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(subscription_cursor_last_fetch_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, subscription_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{subscription_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Subscription Cursor Last Fetch Duration(ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":275,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_eviction_policy{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager eviction policy","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_latest_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_sequence{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager sequence","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":286,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rows sent by remote sink to the Java connector process","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Remote Sink (Java) Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_min_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Kv log store unconsumed min epoch @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(log_store_reader_wait_new_future_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, actor_id, sink_name) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Backpressure @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Backpressure Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, actor_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, sink_id, actor_id, sink_name)","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} @ actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_item_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed item count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_row_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed row count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"kv_log_store_buffer_unconsumed_epoch_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Unconsumed epoch count @ {{sink_id}} {{sink_name}} ({{connector}}) actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Buffer State","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (actor_id, connector, sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, actor_id, connector, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}} ({{connector}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total size of chunks buffered in a barrier","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_sink_chunk_buffer_size{job=~\"$job\",instance=~\"$node\"}) by (sink_id, actor_id, sink_name) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Chunk Buffer Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":304,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka high watermark by source and partition and source latest message by partition, source and actor","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"high watermark: source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest msg: source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka high watermark and source latest message","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":309,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":310,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":311,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":312,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":313,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":314,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":315,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":316,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":317,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":318,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":319,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":320,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":321,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":322,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":323,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":324,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":325,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":326,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":327,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":328,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":329,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":330,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":331,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":332,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":333,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id, sink_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance, sink_id, sink_name)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance, sink_id, sink_name) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":334,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unflushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":335,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":336,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{sink_id}} {{sink_name}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":337,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"}) by (sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write @ {{sink_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_write_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total write","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":338,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(iceberg_read_bytes{job=~\"$job\",instance=~\"$node\"}) by (table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read @ {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(nimtable_read_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total read","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg Read Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":339,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":340,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_retry_count - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":341,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":342,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":343,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":344,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Currently only embedded JS UDF supports this. Others will always show 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":345,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(udf_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_memory_usage - {{name}} {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Memory Usage (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0}
diff --git a/grafana/risingwave-user-dashboard.json b/grafana/risingwave-user-dashboard.json
index 2f19f6e5f6527..80b40dd535b9e 100644
--- a/grafana/risingwave-user-dashboard.json
+++ b/grafana/risingwave-user-dashboard.json
@@ -1 +1 @@
-{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
+{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Actor count per compute node","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"count(actor_info{job=~\"$job\",instance=~\"$node\"}) by (compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Count (Group By Compute Node)","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":5,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n            - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n            - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n            - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n            - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n            - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n            - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n            - Write Stall: Compaction cannot keep up. Stall foreground write.\n            ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, executor_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{executor_name}} (fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{source_name}} (source_id={{source_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, sink_id, sink_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}} @ {{sink_name}} (sink_id={{sink_id}} fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":16,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":19,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0 ","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id)) >=0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}}  {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive  - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":28,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":32,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n                    Objects are classified into 3 groups:\n                    - not referenced by versions: these object are being deleted from object store.\n                    - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n                    - referenced by current version: these objects are in the latest version.\n                    ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_backfill_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Backfill Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":46,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"}],"refresh":"","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0}
diff --git a/integration_tests/kinesis-s3-source/create_mv.sql b/integration_tests/kinesis-s3-source/create_mv.sql
index 7c1cbe1cdeac7..1a5c288035370 100644
--- a/integration_tests/kinesis-s3-source/create_mv.sql
+++ b/integration_tests/kinesis-s3-source/create_mv.sql
@@ -23,6 +23,13 @@ FROM
             ai.ad_id
     ) AS ad_clicks ON ad_impressions.ad_id = ad_clicks.ad_id;
 
+CREATE MATERIALIZED VIEW kinesis_timestamp AS
+SELECT
+    ad_impression._rw_kinesis_timestamp AS timestamp
+FROM
+    ad_impression
+WHERE ad_impression._rw_kinesis_timestamp IS NOT NULL AND ad_impression._rw_kinesis_timestamp >= '2024-01-01 00:00:00+00'::TIMESTAMPTZ;
+
 CREATE MATERIALIZED VIEW ad_ctr_5min AS
 SELECT
     ac.ad_id AS ad_id,
diff --git a/integration_tests/kinesis-s3-source/create_source.sql b/integration_tests/kinesis-s3-source/create_source.sql
index 902eccdf1950b..95f6e2ef933d8 100644
--- a/integration_tests/kinesis-s3-source/create_source.sql
+++ b/integration_tests/kinesis-s3-source/create_source.sql
@@ -2,7 +2,13 @@ CREATE SOURCE ad_impression (
     bid_id BIGINT,
     ad_id BIGINT,
     impression_timestamp TIMESTAMPTZ
-) WITH (
+)
+INCLUDE key
+INCLUDE partition
+INCLUDE offset
+INCLUDE timestamp
+INCLUDE payload
+WITH (
     connector = 'kinesis',
     stream = 'ad-impression',
     aws.region='us-east-1',
diff --git a/integration_tests/kinesis-s3-source/data_check b/integration_tests/kinesis-s3-source/data_check
index b95837955a510..60fd175091c01 100644
--- a/integration_tests/kinesis-s3-source/data_check
+++ b/integration_tests/kinesis-s3-source/data_check
@@ -1 +1 @@
-ad_ctr,ad_ctr_5min
\ No newline at end of file
+ad_ctr,ad_ctr_5min，kinesis_timestamp
\ No newline at end of file
diff --git a/integration_tests/mqtt/create_source.sql b/integration_tests/mqtt/create_source.sql
index 7ebceaa706bcc..bf43ba2d75083 100644
--- a/integration_tests/mqtt/create_source.sql
+++ b/integration_tests/mqtt/create_source.sql
@@ -4,7 +4,7 @@ CREATE TABLE
 CREATE TABLE mqtt_source_table
 (
   id integer,
-  name varchar,
+  name varchar
 )
 WITH (
     connector='mqtt',
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
index 98f0a39a2a3dd..872d6e900c911 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java
@@ -174,6 +174,13 @@ public DbzConnectorConfig(
 
             dbzProps.putAll(mysqlProps);
 
+            if (isCdcSourceJob) {
+                // remove table filtering for the shared MySQL source, since we
+                // allow user to ingest tables in different database
+                LOG.info("Disable table filtering for the shared MySQL source");
+                dbzProps.remove("table.include.list");
+            }
+
         } else if (source == SourceTypeE.POSTGRES) {
             var postgresProps = initiateDbConfig(POSTGRES_CONFIG_FILE, substitutor);
 
@@ -205,6 +212,23 @@ public DbzConnectorConfig(
                 }
             }
 
+            // adapt value of sslmode to the expected value
+            var sslMode = postgresProps.getProperty("database.sslmode");
+            if (sslMode != null) {
+                switch (sslMode) {
+                    case "disabled":
+                        sslMode = "disable";
+                        break;
+                    case "preferred":
+                        sslMode = "prefer";
+                        break;
+                    case "required":
+                        sslMode = "require";
+                        break;
+                }
+                postgresProps.setProperty("database.sslmode", sslMode);
+            }
+
             dbzProps.putAll(postgresProps);
 
             if (isCdcSourceJob) {
diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
index cd7b6d29bb418..00a5f9fc03ad8 100644
--- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
+++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MySqlValidator.java
@@ -15,7 +15,6 @@
 package com.risingwave.connector.source.common;
 
 import com.risingwave.connector.api.TableSchema;
-import com.risingwave.connector.api.source.SourceTypeE;
 import com.risingwave.proto.Data;
 import java.sql.Connection;
 import java.sql.DriverManager;
@@ -45,9 +44,7 @@ public MySqlValidator(
 
         var dbHost = userProps.get(DbzConnectorConfig.HOST);
         var dbPort = userProps.get(DbzConnectorConfig.PORT);
-        var dbName = userProps.get(DbzConnectorConfig.DB_NAME);
-        var jdbcUrl = ValidatorUtils.getJdbcUrl(SourceTypeE.MYSQL, dbHost, dbPort, dbName);
-
+        var jdbcUrl = String.format("jdbc:mysql://%s:%s", dbHost, dbPort);
         var properties = new Properties();
         properties.setProperty("user", userProps.get(DbzConnectorConfig.USER));
         properties.setProperty("password", userProps.get(DbzConnectorConfig.PASSWORD));
@@ -72,6 +69,27 @@ public void validateDbConfig() {
             if ((major > 8) || (major == 8 && minor >= 4)) {
                 throw ValidatorUtils.failedPrecondition("MySQL version should be less than 8.4");
             }
+
+            // "database.name" is a comma-separated list of database names
+            var dbNames = userProps.get(DbzConnectorConfig.DB_NAME);
+            for (var dbName : dbNames.split(",")) {
+                // check the existence of the database
+                try (var stmt =
+                        jdbcConnection.prepareStatement(
+                                ValidatorUtils.getSql("mysql.check_db_exist"))) {
+                    stmt.setString(1, dbName.trim());
+                    var res = stmt.executeQuery();
+                    while (res.next()) {
+                        var ret = res.getInt(1);
+                        if (ret == 0) {
+                            throw ValidatorUtils.invalidArgument(
+                                    String.format(
+                                            "MySQL database '%s' doesn't exist", dbName.trim()));
+                        }
+                    }
+                }
+            }
+
             validateBinlogConfig();
         } catch (SQLException e) {
             throw ValidatorUtils.internalError(e.getMessage());
diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties b/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties
index 04eaf227b65d7..57091adac4eee 100644
--- a/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties
+++ b/java/connector-node/risingwave-connector-service/src/main/resources/validate_sql.properties
@@ -4,6 +4,7 @@ mysql.bin_row_image=show variables like 'binlog_row_image'
 mysql.table=SELECT count(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?
 mysql.table_schema=SELECT COLUMN_NAME, DATA_TYPE, COLUMN_KEY FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION
 mysql.grants=SHOW GRANTS FOR CURRENT_USER()
+mysql.check_db_exist=SELECT count(*) FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = ?
 postgres.wal=show wal_level
 postgres.table=SELECT EXISTS ( SELECT FROM pg_tables WHERE schemaname = ? AND tablename = ?)
 postgres.pk=SELECT a.attname, format_type(a.atttypid, a.atttypmod) AS data_type FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) WHERE i.indrelid = ?::regclass AND i.indisprimary
diff --git a/java/pom.xml b/java/pom.xml
index 7cc8be5a93150..65d6d829c50a1 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -89,7 +89,7 @@
         <sqlite.version>3.45.0.0</sqlite.version>
         <aws.version>2.21.42</aws.version>
         <hive.version>3.1.3</hive.version>
-        <jetty.version>12.0.10</jetty.version>
+        <jetty.version>12.0.12</jetty.version>
         <snappy.version>1.1.10.5</snappy.version>
     </properties>
 
diff --git a/lints/Cargo.lock b/lints/Cargo.lock
index aa1e1e4ef9b32..cdfe23e743f56 100644
--- a/lints/Cargo.lock
+++ b/lints/Cargo.lock
@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "aho-corasick"
@@ -89,9 +89,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.5.0"
+version = "2.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
 
 [[package]]
 name = "block-buffer"
@@ -162,23 +162,22 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clippy_config"
-version = "0.1.81"
-source = "git+https://github.com/risingwavelabs/clippy?rev=5135d0218365e85f3371405b604a7fb1459eb256#5135d0218365e85f3371405b604a7fb1459eb256"
+version = "0.1.83"
+source = "git+https://github.com/risingwavelabs/clippy?rev=755ee9dd292b354fed40dcb9e4cd92d1d604207b#755ee9dd292b354fed40dcb9e4cd92d1d604207b"
 dependencies = [
- "rustc-semver",
+ "itertools",
  "serde",
  "toml 0.7.8",
 ]
 
 [[package]]
 name = "clippy_utils"
-version = "0.1.81"
-source = "git+https://github.com/risingwavelabs/clippy?rev=5135d0218365e85f3371405b604a7fb1459eb256#5135d0218365e85f3371405b604a7fb1459eb256"
+version = "0.1.83"
+source = "git+https://github.com/risingwavelabs/clippy?rev=755ee9dd292b354fed40dcb9e4cd92d1d604207b#755ee9dd292b354fed40dcb9e4cd92d1d604207b"
 dependencies = [
  "arrayvec",
  "clippy_config",
  "itertools",
- "rustc-semver",
  "rustc_apfloat",
 ]
 
@@ -190,9 +189,9 @@ checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
 
 [[package]]
 name = "compiletest_rs"
-version = "0.10.2"
+version = "0.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7225fee1bcf9247bb3a1b1a2d7ecfe2f7a990e549a09d766a257a4ae30dac0d6"
+checksum = "71fcc3c0c91b59c137b3cf8073cbc2f72a49b3d5505660ec88f94da3ed4bb1de"
 dependencies = [
  "diff",
  "filetime",
@@ -207,7 +206,7 @@ dependencies = [
  "serde_derive",
  "serde_json",
  "tester",
- "winapi",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -289,9 +288,9 @@ dependencies = [
 
 [[package]]
 name = "dylint"
-version = "3.1.0"
+version = "3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c6720f18fdd779ad137ab6bc448c11042b1b32eea625ea618c8b953660bba56"
+checksum = "22752a1003c06a3f04f9ea66e4899f97132a1888276fee2d5fbe5f6820eee274"
 dependencies = [
  "ansi_term",
  "anyhow",
@@ -310,13 +309,13 @@ dependencies = [
 
 [[package]]
 name = "dylint_internal"
-version = "3.1.0"
+version = "3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "395dade88bc1a3103ef91b442498943d0072df869736a7c0107a5753c05d006f"
+checksum = "f02176c2fa705973bfce833c3f12a69196959086c776ec967a5aa23b5523d4b6"
 dependencies = [
  "ansi_term",
  "anyhow",
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "cargo_metadata",
  "git2",
  "home",
@@ -333,9 +332,9 @@ dependencies = [
 
 [[package]]
 name = "dylint_linting"
-version = "3.1.0"
+version = "3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d070f934310ccf8f04a940affcce0cd196c1068b6d19c5ae6d975f926968b66"
+checksum = "0338d94b92ffeece90a8ba405c6d051f18b4483558728537a333daa3bb422616"
 dependencies = [
  "cargo_metadata",
  "dylint_internal",
@@ -348,9 +347,9 @@ dependencies = [
 
 [[package]]
 name = "dylint_testing"
-version = "3.1.0"
+version = "3.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7fc8a64c781bde7f1e445dc1205754994012e15a2a6e659ab7166b380c1c22ab"
+checksum = "8e1581603a3fc49b8ede8bed50f5e2d719ce607572f5e5d5d06f200f3285448a"
 dependencies = [
  "anyhow",
  "cargo_metadata",
@@ -411,9 +410,9 @@ dependencies = [
 
 [[package]]
 name = "fastrand"
-version = "2.0.1"
+version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5"
+checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6"
 
 [[package]]
 name = "filetime"
@@ -472,7 +471,7 @@ version = "0.18.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "libc",
  "libgit2-sys",
  "log",
@@ -596,9 +595,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.154"
+version = "0.2.161"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346"
+checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1"
 
 [[package]]
 name = "libgit2-sys"
@@ -620,7 +619,7 @@ version = "0.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "libc",
  "redox_syscall 0.4.1",
 ]
@@ -666,9 +665,9 @@ dependencies = [
 
 [[package]]
 name = "linux-raw-sys"
-version = "0.4.13"
+version = "0.4.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c"
+checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
 
 [[package]]
 name = "log"
@@ -684,11 +683,11 @@ checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167"
 
 [[package]]
 name = "miow"
-version = "0.3.7"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9f1c5b025cda876f66ef43a113f91ebc9f4ccef34843000e0adf6ebbab84e21"
+checksum = "359f76430b20a79f9e20e115b3428614e654f04fab314482fc0fda0ebd3c6044"
 dependencies = [
- "winapi",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]
@@ -703,9 +702,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.19.0"
+version = "1.20.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775"
 
 [[package]]
 name = "openssl-probe"
@@ -757,18 +756,18 @@ checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.79"
+version = "1.0.89"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
+checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.37"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af"
 dependencies = [
  "proc-macro2",
 ]
@@ -804,9 +803,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.10.2"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343"
+checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -816,9 +815,9 @@ dependencies = [
 
 [[package]]
 name = "regex-automata"
-version = "0.4.3"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f"
+checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -827,15 +826,15 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.8.2"
+version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
+checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
 [[package]]
 name = "rust-embed"
-version = "8.4.0"
+version = "8.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19549741604902eb99a7ed0ee177a0663ee1eda51a29f71401f166e47e77806a"
+checksum = "fa66af4a4fdd5e7ebc276f115e895611a34739a9c1c01028383d612d550953c0"
 dependencies = [
  "rust-embed-impl",
  "rust-embed-utils",
@@ -866,12 +865,6 @@ dependencies = [
  "walkdir",
 ]
 
-[[package]]
-name = "rustc-semver"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5be1bdc7edf596692617627bbfeaba522131b18e06ca4df2b6b689e3c5d5ce84"
-
 [[package]]
 name = "rustc_apfloat"
 version = "0.2.1+llvm-462a31f5a5ab"
@@ -884,23 +877,23 @@ dependencies = [
 
 [[package]]
 name = "rustfix"
-version = "0.6.1"
+version = "0.8.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ecd2853d9e26988467753bd9912c3a126f642d05d229a4b53f5752ee36c56481"
+checksum = "8fb2b066405a6d48a1b39c0022270503e352ae84da0c24e1d5f8ffc38e97a325"
 dependencies = [
- "anyhow",
- "log",
  "serde",
  "serde_json",
+ "thiserror",
+ "tracing",
 ]
 
 [[package]]
 name = "rustix"
-version = "0.38.34"
+version = "0.38.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a"
 dependencies = [
- "bitflags 2.5.0",
+ "bitflags 2.6.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -939,18 +932,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.193"
+version = "1.0.214"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89"
+checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.193"
+version = "1.0.214"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3"
+checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -959,11 +952,12 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.108"
+version = "1.0.132"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b"
+checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03"
 dependencies = [
  "itoa",
+ "memchr",
  "ryu",
  "serde",
 ]
@@ -996,9 +990,9 @@ checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
 
 [[package]]
 name = "syn"
-version = "2.0.39"
+version = "2.0.85"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a"
+checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1007,14 +1001,15 @@ dependencies = [
 
 [[package]]
 name = "tempfile"
-version = "3.10.1"
+version = "3.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b"
 dependencies = [
  "cfg-if",
  "fastrand",
+ "once_cell",
  "rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -1043,9 +1038,9 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.50"
+version = "1.0.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2"
+checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5"
 dependencies = [
  "thiserror-impl",
 ]
@@ -1074,9 +1069,9 @@ dependencies = [
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.50"
+version = "1.0.65"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
+checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1312,7 +1307,16 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
 dependencies = [
- "windows-targets 0.52.0",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -1332,17 +1336,18 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.0",
- "windows_aarch64_msvc 0.52.0",
- "windows_i686_gnu 0.52.0",
- "windows_i686_msvc 0.52.0",
- "windows_x86_64_gnu 0.52.0",
- "windows_x86_64_gnullvm 0.52.0",
- "windows_x86_64_msvc 0.52.0",
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
 ]
 
 [[package]]
@@ -1353,9 +1358,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -1365,9 +1370,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -1377,9 +1382,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.0"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -1389,9 +1400,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -1401,9 +1412,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
@@ -1413,9 +1424,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -1425,9 +1436,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.0"
+version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
 [[package]]
 name = "winnow"
diff --git a/lints/Cargo.toml b/lints/Cargo.toml
index e0b8fe5d96664..40c90cda3d86d 100644
--- a/lints/Cargo.toml
+++ b/lints/Cargo.toml
@@ -14,12 +14,12 @@ path = "ui/format_error.rs"
 # See `README.md` before bumping the version.
 # Remember to update the version in `ci/Dockerfile` as well.
 [dependencies]
-clippy_utils = { git = "https://github.com/risingwavelabs/clippy", rev = "61e1d2fd7062e46ccf1237707ee6da5aac018f70" }
-dylint_linting = "3.1.0"
+clippy_utils = { git = "https://github.com/risingwavelabs/clippy", rev = "755ee9dd292b354fed40dcb9e4cd92d1d604207b" }
+dylint_linting = "3.2.1"
 itertools = "0.12"
 
 [dev-dependencies]
-dylint_testing = "3.1.0"
+dylint_testing = "3.2.1"
 
 # UI test dependencies
 anyhow = "1"
diff --git a/lints/rust-toolchain b/lints/rust-toolchain
index 31dbc57d04b2b..c4f89bf3f7be5 100644
--- a/lints/rust-toolchain
+++ b/lints/rust-toolchain
@@ -1,5 +1,5 @@
 # See `README.md` before bumping the version.
 
 [toolchain]
-channel = "nightly-2024-07-19"
+channel = "nightly-2024-10-11"
 components = ["llvm-tools-preview", "rustc-dev"]
diff --git a/lints/src/format_error.rs b/lints/src/format_error.rs
index 402adc4aa5af0..b4304aad82cdc 100644
--- a/lints/src/format_error.rs
+++ b/lints/src/format_error.rs
@@ -104,7 +104,7 @@ impl<'tcx> LateLintPass<'tcx> for FormatError {
                     if let FormatArgsPiece::Placeholder(placeholder) = piece
                         && let Ok(index) = placeholder.argument.index
                         && let Some(arg) = format_args.arguments.all_args().get(index)
-                        && let Ok(arg_expr) = find_format_arg_expr(expr, arg)
+                        && let Some(arg_expr) = find_format_arg_expr(expr, arg)
                     {
                         if in_tracing_event_macro {
                             check_fmt_arg_in_tracing_event(cx, arg_expr);
diff --git a/proto/batch_plan.proto b/proto/batch_plan.proto
index aefa17de23700..b46230b2438d6 100644
--- a/proto/batch_plan.proto
+++ b/proto/batch_plan.proto
@@ -104,6 +104,17 @@ message PostgresQueryNode {
   string query = 7;
 }
 
+// NOTE(kwannoel): This will only be used in batch mode. We can change the definition as needed.
+message MySqlQueryNode {
+  repeated plan_common.ColumnDesc columns = 1;
+  string hostname = 2;
+  string port = 3;
+  string username = 4;
+  string password = 5;
+  string database = 6;
+  string query = 7;
+}
+
 message ProjectNode {
   repeated expr.ExprNode select_list = 1;
 }
@@ -122,6 +133,7 @@ message LogRowSeqScanNode {
   common.Buffer vnode_bitmap = 3;
   common.BatchQueryEpoch old_epoch = 4;
   common.BatchQueryEpoch new_epoch = 5;
+  bool ordered = 6;
 }
 
 message InsertNode {
@@ -385,6 +397,7 @@ message PlanNode {
     FileScanNode file_scan = 38;
     IcebergScanNode iceberg_scan = 39;
     PostgresQueryNode postgres_query = 40;
+    MySqlQueryNode mysql_query = 41;
     // The following nodes are used for testing.
     bool block_executor = 100;
     bool busy_loop_executor = 101;
diff --git a/proto/catalog.proto b/proto/catalog.proto
index f792eccc0cab6..5383104e9c0f2 100644
--- a/proto/catalog.proto
+++ b/proto/catalog.proto
@@ -95,6 +95,8 @@ message StreamSourceInfo {
 }
 
 message Source {
+  // For shared source, this is the same as the job id.
+  // For non-shared source and table with connector, this is a different oid.
   uint32 id = 1;
   uint32 schema_id = 2;
   uint32 database_id = 3;
@@ -243,7 +245,7 @@ message Connection {
   uint32 database_id = 3;
   string name = 4;
   oneof info {
-    PrivateLinkService private_link_service = 5;
+    PrivateLinkService private_link_service = 5 [deprecated = true];
   }
   uint32 owner = 6;
 }
@@ -419,13 +421,13 @@ message Table {
 
   // Total vnode count of the table.
   //
-  // Can be unset if...
-  // - The catalog is generated by the frontend and not yet persisted, this is
-  //   because the vnode count of each fragment (then internal tables) is determined
-  //   by the meta service.
-  // - The table is created in older versions where variable vnode count is not
-  //   supported, in which case a default value of 256 should be used.
-  //   Use `VnodeCountCompat::vnode_count` to access it.
+  // Use `VnodeCountCompat::vnode_count` to access it.
+  //
+  // - Can be unset if the table is created in older versions where variable vnode count is not
+  //   supported, in which case a default value of 256 (or 1 for singleton) should be used.
+  // - Can be placeholder value `Some(0)` if the catalog is generated by the frontend and the
+  //   corresponding job is still in `Creating` status, in which case calling `vnode_count`
+  //   will panic.
   //
   // Please note that this field is not intended to describe the expected vnode count
   // for a streaming job. Instead, refer to `stream_plan.StreamFragmentGraph.max_parallelism`.
diff --git a/proto/ddl_service.proto b/proto/ddl_service.proto
index de860593e8105..6467bd6e1d7e7 100644
--- a/proto/ddl_service.proto
+++ b/proto/ddl_service.proto
@@ -268,6 +268,26 @@ message AlterOwnerResponse {
   WaitVersion version = 2;
 }
 
+message AlterSwapRenameRequest {
+  message ObjectNameSwapPair {
+    uint32 src_object_id = 1;
+    uint32 dst_object_id = 2;
+  }
+  oneof object {
+    ObjectNameSwapPair schema = 1;
+    ObjectNameSwapPair table = 2;
+    ObjectNameSwapPair view = 3;
+    ObjectNameSwapPair source = 4;
+    ObjectNameSwapPair sink = 5;
+    ObjectNameSwapPair subscription = 6;
+  }
+}
+
+message AlterSwapRenameResponse {
+  common.Status status = 1;
+  WaitVersion version = 2;
+}
+
 message CreateFunctionRequest {
   catalog.Function function = 1;
 }
@@ -513,4 +533,5 @@ service DdlService {
   rpc Wait(WaitRequest) returns (WaitResponse);
   rpc CommentOn(CommentOnRequest) returns (CommentOnResponse);
   rpc AutoSchemaChange(AutoSchemaChangeRequest) returns (AutoSchemaChangeResponse);
+  rpc AlterSwapRename(AlterSwapRenameRequest) returns (AlterSwapRenameResponse);
 }
diff --git a/proto/expr.proto b/proto/expr.proto
index 43e2002933a47..5330843512849 100644
--- a/proto/expr.proto
+++ b/proto/expr.proto
@@ -365,6 +365,8 @@ message TableFunction {
     FILE_SCAN = 19;
     // postgres query
     POSTGRES_QUERY = 20;
+    // mysql query
+    MYSQL_QUERY = 21;
     // User defined table function
     USER_DEFINED = 100;
   }
diff --git a/proto/hummock.proto b/proto/hummock.proto
index c59a181197f9e..ec0e20f6013ce 100644
--- a/proto/hummock.proto
+++ b/proto/hummock.proto
@@ -63,6 +63,10 @@ message InputLevel {
   repeated SstableInfo table_infos = 3;
 }
 
+message NewL0SubLevel {
+  repeated SstableInfo inserted_table_infos = 1;
+}
+
 message IntraLevelDelta {
   uint32 level_idx = 1;
   uint64 l0_sub_level_id = 2;
@@ -112,6 +116,7 @@ message GroupDelta {
     GroupConstruct group_construct = 2;
     GroupDestroy group_destroy = 3;
     GroupMerge group_merge = 6;
+    NewL0SubLevel new_l0_sub_level = 7;
   }
 }
 
diff --git a/proto/meta.proto b/proto/meta.proto
index d51e22d444502..b6c9521ad673c 100644
--- a/proto/meta.proto
+++ b/proto/meta.proto
@@ -92,7 +92,7 @@ message TableFragments {
     // Duplicated from the length of the vnode bitmap in any actor of the fragment.
     //
     // Can be unset if the fragment is created in older versions where variable vnode count is not
-    // supported, in which case a default value of 256 should be used.
+    // supported, in which case a default value of 256 (or 1 for singleton) should be used.
     // Use `VnodeCountCompat::vnode_count` to access it.
     optional uint32 maybe_vnode_count = 8;
   }
@@ -156,7 +156,7 @@ message MigrationPlan {
 }
 
 message FlushRequest {
-  bool checkpoint = 1;
+  uint32 database_id = 1;
 }
 
 message FlushResponse {
@@ -769,6 +769,12 @@ message EventLog {
     string cdc_table_id = 3;
     string upstream_ddl = 4;
   }
+  message EventSinkFail {
+    uint32 sink_id = 1;
+    string sink_name = 2;
+    string connector = 3;
+    string error = 4;
+  }
   // Event logs identifier, which should be populated by event log service.
   optional string unique_id = 1;
   // Processing time, which should be populated by event log service.
@@ -782,6 +788,7 @@ message EventLog {
     EventCollectBarrierFail collect_barrier_fail = 8;
     EventLog.EventWorkerNodePanic worker_node_panic = 9;
     EventLog.EventAutoSchemaChangeFail auto_schema_change_fail = 10;
+    EventLog.EventSinkFail sink_fail = 11;
   }
 }
 
@@ -795,6 +802,7 @@ message AddEventLogRequest {
   // A subset of EventLog.event that can be added by non meta node.
   oneof event {
     EventLog.EventWorkerNodePanic worker_node_panic = 1;
+    EventLog.EventSinkFail sink_fail = 2;
   }
 }
 
diff --git a/proto/plan_common.proto b/proto/plan_common.proto
index f561ee427ea46..487ab54e2a666 100644
--- a/proto/plan_common.proto
+++ b/proto/plan_common.proto
@@ -100,7 +100,7 @@ message StorageTableDesc {
   // Total vnode count of the table.
   //
   // Can be unset if the table is created in older versions where variable vnode count is not
-  // supported, in which case a default value of 256 should be used.
+  // supported, in which case a default value of 256 (or 1 for singleton) should be used.
   // Use `VnodeCountCompat::vnode_count` to access it.
   optional uint32 maybe_vnode_count = 12;
 }
diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto
index fe3a2c93d11ab..ad0601189ec9f 100644
--- a/proto/stream_plan.proto
+++ b/proto/stream_plan.proto
@@ -491,6 +491,8 @@ message TemporalJoinNode {
   repeated uint32 table_output_indices = 8;
   // The state table used for non-append-only temporal join.
   optional catalog.Table memo_table = 9;
+  // If it is a nested lool temporal join
+  bool is_nested_loop = 10;
 }
 
 message DynamicFilterNode {
diff --git a/proto/stream_service.proto b/proto/stream_service.proto
index e98d1fdaed754..011c06284397a 100644
--- a/proto/stream_service.proto
+++ b/proto/stream_service.proto
@@ -24,10 +24,15 @@ message InjectBarrierRequest {
 
 message BarrierCompleteResponse {
   message CreateMviewProgress {
+    // Note: ideally we should use `executor_id`, but `actor_id` is ok-ish.
+    // See <https://github.com/risingwavelabs/risingwave/issues/6236>.
     uint32 backfill_actor_id = 1;
     bool done = 2;
+    // MV backfill snapshot read epoch (0 for Done / Source backfill)
     uint64 consumed_epoch = 3;
+    // MV backfill snapshot read rows / Source backfilled rows
     uint64 consumed_rows = 4;
+    uint32 pending_barrier_num = 5;
   }
   string request_id = 1;
   common.Status status = 2;
diff --git a/risedev.yml b/risedev.yml
index 4fca6ebd73488..8e3668dcb49c2 100644
--- a/risedev.yml
+++ b/risedev.yml
@@ -43,7 +43,7 @@ profile:
       # - use: grafana     # visualization
 
       - use: meta-node
-      #   - meta-backend: postgres
+        # meta-backend: postgres
       - use: compute-node
       - use: frontend
 
@@ -584,6 +584,28 @@ profile:
       - use: frontend
       - use: compactor
 
+  ci-backfill-3cn-1fe:
+    config-path: src/config/ci-longer-streaming-upload-timeout.toml
+    steps:
+      - use: minio
+      - use: sqlite
+      - use: meta-node
+        meta-backend: sqlite
+      - use: compute-node
+        port: 5687
+        exporter-port: 1222
+        enable-tiered-cache: true
+      - use: compute-node
+        port: 5688
+        exporter-port: 1223
+        enable-tiered-cache: true
+      - use: compute-node
+        port: 5689
+        exporter-port: 1224
+        enable-tiered-cache: true
+      - use: frontend
+      - use: compactor
+
   ci-backfill-3cn-1fe-with-monitoring:
     config-path: src/config/ci-longer-streaming-upload-timeout.toml
     steps:
@@ -801,6 +823,122 @@ profile:
         health-check-port: 6788
       - use: compactor
 
+  ci-3streaming-2serving-3fe-pg-backend:
+    config-path: src/config/ci.toml
+    steps:
+      - use: minio
+      - use: postgres
+        port: 5432
+        address: db
+        database: metadata
+        user: postgres
+        password: postgres
+        user-managed: true
+        application: metastore
+      - use: meta-node
+        meta-backend: postgres
+      - use: compute-node
+        port: 5687
+        exporter-port: 1222
+        enable-tiered-cache: true
+        role: streaming
+        parallelism: 4
+      - use: compute-node
+        port: 5688
+        exporter-port: 1223
+        enable-tiered-cache: true
+        role: streaming
+        parallelism: 4
+      - use: compute-node
+        port: 5689
+        exporter-port: 1224
+        enable-tiered-cache: true
+        role: streaming
+        parallelism: 4
+      - use: compute-node
+        port: 5685
+        exporter-port: 1225
+        enable-tiered-cache: true
+        role: serving
+        parallelism: 4
+      - use: compute-node
+        port: 5686
+        exporter-port: 1226
+        enable-tiered-cache: true
+        role: serving
+        parallelism: 8
+      - use: frontend
+        port: 4565
+        exporter-port: 2222
+        health-check-port: 6786
+      - use: frontend
+        port: 4566
+        exporter-port: 2223
+        health-check-port: 6787
+      - use: frontend
+        port: 4567
+        exporter-port: 2224
+        health-check-port: 6788
+      - use: compactor
+
+  ci-3streaming-2serving-3fe-mysql-backend:
+    config-path: src/config/ci.toml
+    steps:
+      - use: minio
+      - use: mysql
+        port: 3306
+        address: mysql
+        database: metadata
+        user: root
+        password: 123456
+        user-managed: true
+        application: metastore
+      - use: meta-node
+        meta-backend: mysql
+      - use: compute-node
+        port: 5687
+        exporter-port: 1222
+        enable-tiered-cache: true
+        role: streaming
+        parallelism: 4
+      - use: compute-node
+        port: 5688
+        exporter-port: 1223
+        enable-tiered-cache: true
+        role: streaming
+        parallelism: 4
+      - use: compute-node
+        port: 5689
+        exporter-port: 1224
+        enable-tiered-cache: true
+        role: streaming
+        parallelism: 4
+      - use: compute-node
+        port: 5685
+        exporter-port: 1225
+        enable-tiered-cache: true
+        role: serving
+        parallelism: 4
+      - use: compute-node
+        port: 5686
+        exporter-port: 1226
+        enable-tiered-cache: true
+        role: serving
+        parallelism: 8
+      - use: frontend
+        port: 4565
+        exporter-port: 2222
+        health-check-port: 6786
+      - use: frontend
+        port: 4566
+        exporter-port: 2223
+        health-check-port: 6787
+      - use: frontend
+        port: 4567
+        exporter-port: 2224
+        health-check-port: 6788
+      - use: compactor
+
   ci-kafka:
     config-path: src/config/ci.toml
     steps:
@@ -821,6 +959,25 @@ profile:
         address: schemaregistry
         port: 8082
 
+  local-inline-source-test:
+    config-path: src/config/ci-recovery.toml
+    steps:
+      - use: minio
+      - use: sqlite
+      - use: meta-node
+        meta-backend: sqlite
+      - use: compute-node
+        enable-tiered-cache: true
+      - use: frontend
+      - use: compactor
+      - use: pubsub
+        persist-data: true
+      - use: kafka
+        persist-data: true
+      - use: schema-registry
+      - use: mysql
+      - use: postgres
+
   ci-inline-source-test:
     config-path: src/config/ci-recovery.toml
     steps:
@@ -848,6 +1005,12 @@ profile:
         user: root
         password: 123456
         user-managed: true
+      - use: postgres
+        port: 5432
+        address: db
+        user: postgres
+        password: postgres
+        user-managed: true
 
   ci-redis:
     config-path: src/config/ci.toml
diff --git a/src/batch/Cargo.toml b/src/batch/Cargo.toml
index 46c4aa7b9de6e..ee6f757e17376 100644
--- a/src/batch/Cargo.toml
+++ b/src/batch/Cargo.toml
@@ -19,6 +19,7 @@ assert_matches = "1"
 async-recursion = "1"
 async-trait = "0.1"
 bytes = "1"
+chrono = "0.4"
 either = "1"
 foyer = { workspace = true }
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
@@ -29,6 +30,7 @@ hytra = "0.1.2"
 iceberg = { workspace = true }
 itertools = { workspace = true }
 memcomparable = "0.2"
+mysql_async = { workspace = true }
 opendal = { workspace = true }
 parking_lot = { workspace = true }
 parquet = { workspace = true }
@@ -45,6 +47,7 @@ risingwave_hummock_sdk = { workspace = true }
 risingwave_pb = { workspace = true }
 risingwave_rpc_client = { workspace = true }
 risingwave_storage = { workspace = true }
+rust_decimal = "1"
 rw_futures_util = { workspace = true }
 scopeguard = "1"
 serde_json = "1"
diff --git a/src/batch/src/error.rs b/src/batch/src/error.rs
index ae26fca228321..b0723277a6f5e 100644
--- a/src/batch/src/error.rs
+++ b/src/batch/src/error.rs
@@ -17,9 +17,11 @@
 use std::sync::Arc;
 
 pub use anyhow::anyhow;
+use iceberg::Error as IcebergError;
+use mysql_async::Error as MySqlError;
 use parquet::errors::ParquetError;
 use risingwave_common::array::ArrayError;
-use risingwave_common::error::BoxedError;
+use risingwave_common::error::{def_anyhow_newtype, def_anyhow_variant, BoxedError};
 use risingwave_common::util::value_encoding::error::ValueEncodingError;
 use risingwave_connector::error::ConnectorError;
 use risingwave_dml::error::DmlError;
@@ -29,7 +31,7 @@ use risingwave_rpc_client::error::{RpcError, ToTonicStatus};
 use risingwave_storage::error::StorageError;
 use thiserror::Error;
 use thiserror_ext::Construct;
-use tokio_postgres;
+use tokio_postgres::Error as PostgresError;
 use tonic::Status;
 
 use crate::worker_manager::worker_node_manager::FragmentId;
@@ -114,25 +116,11 @@ pub enum BatchError {
         DmlError,
     ),
 
-    #[error(transparent)]
-    Iceberg(
-        #[from]
-        #[backtrace]
-        iceberg::Error,
-    ),
-
-    #[error(transparent)]
-    Parquet(
-        #[from]
-        #[backtrace]
-        ParquetError,
-    ),
-
-    #[error(transparent)]
-    Postgres(
+    #[error("External system error: {0}")]
+    ExternalSystemError(
         #[from]
         #[backtrace]
-        tokio_postgres::Error,
+        BatchExternalSystemError,
     ),
 
     // Make the ref-counted type to be a variant for easier code structuring.
@@ -200,3 +188,14 @@ impl From<ConnectorError> for BatchError {
         Self::Connector(value.into())
     }
 }
+
+// Define a external system error
+def_anyhow_variant! {
+    pub BatchExternalSystemError,
+    BatchError ExternalSystemError,
+
+    PostgresError => "Postgres error",
+    IcebergError => "Iceberg error",
+    ParquetError => "Parquet error",
+    MySqlError => "MySQL error",
+}
diff --git a/src/batch/src/executor/group_top_n.rs b/src/batch/src/executor/group_top_n.rs
index b1f4791358131..7dda468066e93 100644
--- a/src/batch/src/executor/group_top_n.rs
+++ b/src/batch/src/executor/group_top_n.rs
@@ -20,6 +20,7 @@ use futures_async_stream::try_stream;
 use hashbrown::HashMap;
 use itertools::Itertools;
 use risingwave_common::array::DataChunk;
+use risingwave_common::bitmap::FilterByBitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::hash::{HashKey, HashKeyDispatcher, PrecomputedBuildHasher};
 use risingwave_common::memory::{MemoryContext, MonitoredGlobalAlloc};
@@ -198,16 +199,12 @@ impl<K: HashKey> GroupTopNExecutor<K> {
             let chunk = Arc::new(chunk?);
             let keys = K::build_many(self.group_key.as_slice(), &chunk);
 
-            for (row_id, ((encoded_row, key), visible)) in
-                encode_chunk(&chunk, &self.column_orders)?
-                    .into_iter()
-                    .zip_eq_fast(keys.into_iter())
-                    .zip_eq_fast(chunk.visibility().iter())
-                    .enumerate()
+            for (row_id, (encoded_row, key)) in encode_chunk(&chunk, &self.column_orders)?
+                .into_iter()
+                .zip_eq_fast(keys.into_iter())
+                .enumerate()
+                .filter_by_bitmap(chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 let heap = groups.entry(key).or_insert_with(|| {
                     TopNHeap::new(
                         self.limit,
diff --git a/src/batch/src/executor/hash_agg.rs b/src/batch/src/executor/hash_agg.rs
index bde5d36bd8c64..ea780d7bf83b3 100644
--- a/src/batch/src/executor/hash_agg.rs
+++ b/src/batch/src/executor/hash_agg.rs
@@ -21,7 +21,7 @@ use futures_async_stream::try_stream;
 use hashbrown::hash_map::Entry;
 use itertools::Itertools;
 use risingwave_common::array::{DataChunk, StreamChunk};
-use risingwave_common::bitmap::Bitmap;
+use risingwave_common::bitmap::{Bitmap, FilterByBitmap};
 use risingwave_common::catalog::{Field, Schema};
 use risingwave_common::hash::{HashKey, HashKeyDispatcher, PrecomputedBuildHasher};
 use risingwave_common::memory::MemoryContext;
@@ -545,14 +545,11 @@ impl<K: HashKey + Send + Sync> HashAggExecutor<K> {
             let chunk = StreamChunk::from(chunk?);
             let keys = K::build_many(self.group_key_columns.as_slice(), &chunk);
             let mut memory_usage_diff = 0;
-            for (row_id, (key, visible)) in keys
+            for (row_id, key) in keys
                 .into_iter()
-                .zip_eq_fast(chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 let mut new_group = false;
                 let states = match groups.entry(key) {
                     Entry::Occupied(entry) => entry.into_mut(),
diff --git a/src/batch/src/executor/iceberg_scan.rs b/src/batch/src/executor/iceberg_scan.rs
index f1da314c1003e..229cff77e9b9c 100644
--- a/src/batch/src/executor/iceberg_scan.rs
+++ b/src/batch/src/executor/iceberg_scan.rs
@@ -151,13 +151,10 @@ impl IcebergScanExecutor {
                 .build();
             let file_scan_stream = tokio_stream::once(Ok(data_file_scan_task));
 
-            let mut record_batch_stream = reader
-                .read(Box::pin(file_scan_stream))
-                .map_err(BatchError::Iceberg)?
-                .enumerate();
+            let mut record_batch_stream = reader.read(Box::pin(file_scan_stream))?.enumerate();
 
             while let Some((index, record_batch)) = record_batch_stream.next().await {
-                let record_batch = record_batch.map_err(BatchError::Iceberg)?;
+                let record_batch = record_batch?;
 
                 let chunk = IcebergArrowConvert.chunk_from_record_batch(&record_batch)?;
                 // position delete
@@ -195,8 +192,7 @@ impl BoxedExecutorBuilder for IcebergScanExecutorBuilder {
             source_node.with_properties.clone(),
             source_node.secret_refs.clone(),
         );
-        let config = ConnectorProperties::extract(options_with_secret.clone(), false)
-            .map_err(BatchError::connector)?;
+        let config = ConnectorProperties::extract(options_with_secret.clone(), false)?;
 
         let split_list = source_node
             .split
@@ -279,12 +275,10 @@ impl PositionDeleteFilter {
 
         let reader = table.reader_builder().with_batch_size(batch_size).build();
 
-        let mut record_batch_stream = reader
-            .read(Box::pin(position_delete_file_scan_stream))
-            .map_err(BatchError::Iceberg)?;
+        let mut record_batch_stream = reader.read(Box::pin(position_delete_file_scan_stream))?;
 
         while let Some(record_batch) = record_batch_stream.next().await {
-            let record_batch = record_batch.map_err(BatchError::Iceberg)?;
+            let record_batch = record_batch?;
             let chunk = IcebergArrowConvert.chunk_from_record_batch(&record_batch)?;
             for row in chunk.rows() {
                 // The schema is fixed. `0` must be `file_path`, `1` must be `pos`.
@@ -382,12 +376,10 @@ impl EqualityDeleteFilter {
             let reader = table.reader_builder().with_batch_size(batch_size).build();
             let delete_file_scan_stream = tokio_stream::once(Ok(equality_delete_file_scan_task));
 
-            let mut delete_record_batch_stream = reader
-                .read(Box::pin(delete_file_scan_stream))
-                .map_err(BatchError::Iceberg)?;
+            let mut delete_record_batch_stream = reader.read(Box::pin(delete_file_scan_stream))?;
 
             while let Some(record_batch) = delete_record_batch_stream.next().await {
-                let record_batch = record_batch.map_err(BatchError::Iceberg)?;
+                let record_batch = record_batch?;
 
                 let chunk = IcebergArrowConvert.chunk_from_record_batch(&record_batch)?;
                 for row in chunk.rows() {
diff --git a/src/batch/src/executor/join/chunked_data.rs b/src/batch/src/executor/join/chunked_data.rs
index 158f9cba9f289..be947a885cb58 100644
--- a/src/batch/src/executor/join/chunked_data.rs
+++ b/src/batch/src/executor/join/chunked_data.rs
@@ -37,7 +37,7 @@ pub(super) struct AllRowIdIter<'a> {
     chunk_offsets: &'a [usize],
 }
 
-impl<'a> Iterator for AllRowIdIter<'a> {
+impl Iterator for AllRowIdIter<'_> {
     type Item = RowId;
 
     fn next(&mut self) -> Option<Self::Item> {
diff --git a/src/batch/src/executor/join/hash_join.rs b/src/batch/src/executor/join/hash_join.rs
index 863e53035626a..5ec115e0990a2 100644
--- a/src/batch/src/executor/join/hash_join.rs
+++ b/src/batch/src/executor/join/hash_join.rs
@@ -21,7 +21,7 @@ use bytes::Bytes;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
 use risingwave_common::array::{Array, DataChunk, RowRef};
-use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
+use risingwave_common::bitmap::{Bitmap, BitmapBuilder, FilterByBitmap};
 use risingwave_common::catalog::Schema;
 use risingwave_common::hash::{HashKey, HashKeyDispatcher, PrecomputedBuildHasher};
 use risingwave_common::memory::{MemoryContext, MonitoredGlobalAlloc};
@@ -158,7 +158,7 @@ impl ChunkedData<Option<RowId>> {
     }
 }
 
-impl<'a> Iterator for RowIdIter<'a> {
+impl Iterator for RowIdIter<'_> {
     type Item = RowId;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -514,15 +514,12 @@ impl<K: HashKey> HashJoinExecutor<K> {
             for (build_chunk_id, build_chunk) in build_side.iter().enumerate() {
                 let build_keys = K::build_many(&self.build_key_idxs, build_chunk);
 
-                for (build_row_id, (build_key, visible)) in build_keys
+                for (build_row_id, build_key) in build_keys
                     .into_iter()
-                    .zip_eq_fast(build_chunk.visibility().iter())
                     .enumerate()
+                    .filter_by_bitmap(build_chunk.visibility())
                 {
                     self.shutdown_rx.check()?;
-                    if !visible {
-                        continue;
-                    }
                     // Only insert key to hash map if it is consistent with the null safe restriction.
                     if build_key.null_bitmap().is_subset(&null_matched) {
                         let row_id = RowId::new(build_chunk_id, build_row_id);
@@ -765,14 +762,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 for build_row_id in
                     next_build_row_with_same_key.row_id_iter(hash_map.get(probe_key).copied())
                 {
@@ -828,14 +822,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 if let Some(first_matched_build_row_id) = hash_map.get(probe_key) {
                     for build_row_id in
                         next_build_row_with_same_key.row_id_iter(Some(*first_matched_build_row_id))
@@ -898,14 +889,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 if let Some(first_matched_build_row_id) = hash_map.get(probe_key) {
                     non_equi_state
                         .first_output_row_id
@@ -979,14 +967,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 shutdown_rx.check()?;
                 if !ANTI_JOIN {
                     if hash_map.contains_key(probe_key) {
@@ -1043,14 +1028,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 non_equi_state.found_matched = false;
                 if let Some(first_matched_build_row_id) = hash_map.get(probe_key) {
                     non_equi_state
@@ -1119,14 +1101,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 non_equi_state.found_matched = false;
                 if let Some(first_matched_build_row_id) = hash_map.get(probe_key) {
                     non_equi_state
@@ -1201,14 +1180,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 for build_row_id in
                     next_build_row_with_same_key.row_id_iter(hash_map.get(probe_key).copied())
                 {
@@ -1266,14 +1242,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 for build_row_id in
                     next_build_row_with_same_key.row_id_iter(hash_map.get(probe_key).copied())
                 {
@@ -1338,13 +1311,7 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_key, visible) in probe_keys
-                .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
-            {
-                if !visible {
-                    continue;
-                }
+            for probe_key in probe_keys.iter().filter_by_bitmap(probe_chunk.visibility()) {
                 for build_row_id in
                     next_build_row_with_same_key.row_id_iter(hash_map.get(probe_key).copied())
                 {
@@ -1392,14 +1359,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 for build_row_id in
                     next_build_row_with_same_key.row_id_iter(hash_map.get(probe_key).copied())
                 {
@@ -1465,14 +1429,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 if let Some(first_matched_build_row_id) = hash_map.get(probe_key) {
                     for build_row_id in
                         next_build_row_with_same_key.row_id_iter(Some(*first_matched_build_row_id))
@@ -1547,14 +1508,11 @@ impl<K: HashKey> HashJoinExecutor<K> {
         for probe_chunk in probe_side.execute() {
             let probe_chunk = probe_chunk?;
             let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk);
-            for (probe_row_id, (probe_key, visible)) in probe_keys
+            for (probe_row_id, probe_key) in probe_keys
                 .iter()
-                .zip_eq_fast(probe_chunk.visibility().iter())
                 .enumerate()
+                .filter_by_bitmap(probe_chunk.visibility())
             {
-                if !visible {
-                    continue;
-                }
                 left_non_equi_state.found_matched = false;
                 if let Some(first_matched_build_row_id) = hash_map.get(probe_key) {
                     left_non_equi_state
diff --git a/src/batch/src/executor/join/lookup_join_base.rs b/src/batch/src/executor/join/lookup_join_base.rs
index 39a39b9a1424d..743ae25cf4d6a 100644
--- a/src/batch/src/executor/join/lookup_join_base.rs
+++ b/src/batch/src/executor/join/lookup_join_base.rs
@@ -18,13 +18,13 @@ use futures::StreamExt;
 use futures_async_stream::try_stream;
 use itertools::Itertools;
 use risingwave_common::array::DataChunk;
+use risingwave_common::bitmap::FilterByBitmap;
 use risingwave_common::catalog::Schema;
 use risingwave_common::hash::{HashKey, NullBitmap, PrecomputedBuildHasher};
 use risingwave_common::memory::MemoryContext;
 use risingwave_common::row::Row;
 use risingwave_common::types::{DataType, ToOwnedDatum};
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
-use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::sort_util::{cmp_datum_iter, OrderType};
 use risingwave_common_estimate_size::EstimateSize;
 use risingwave_expr::expr::BoxedExpression;
@@ -150,14 +150,11 @@ impl<K: HashKey> LookupJoinBase<K> {
             for (build_chunk_id, build_chunk) in build_side.iter().enumerate() {
                 let build_keys = K::build_many(&hash_join_build_side_key_idxs, build_chunk);
 
-                for (build_row_id, (build_key, visible)) in build_keys
+                for (build_row_id, build_key) in build_keys
                     .into_iter()
-                    .zip_eq_fast(build_chunk.visibility().iter())
                     .enumerate()
+                    .filter_by_bitmap(build_chunk.visibility())
                 {
-                    if !visible {
-                        continue;
-                    }
                     // Only insert key to hash map if it is consistent with the null safe
                     // restriction.
                     if build_key.null_bitmap().is_subset(&null_matched) {
diff --git a/src/batch/src/executor/log_row_seq_scan.rs b/src/batch/src/executor/log_row_seq_scan.rs
index 5b88736fb06f6..6f40f42fbba8c 100644
--- a/src/batch/src/executor/log_row_seq_scan.rs
+++ b/src/batch/src/executor/log_row_seq_scan.rs
@@ -52,6 +52,7 @@ pub struct LogRowSeqScanExecutor<S: StateStore> {
     old_epoch: u64,
     new_epoch: u64,
     version_id: HummockVersionId,
+    ordered: bool,
 }
 
 impl<S: StateStore> LogRowSeqScanExecutor<S> {
@@ -63,6 +64,7 @@ impl<S: StateStore> LogRowSeqScanExecutor<S> {
         chunk_size: usize,
         identity: String,
         metrics: Option<BatchMetrics>,
+        ordered: bool,
     ) -> Self {
         let mut schema = table.schema().clone();
         schema.fields.push(Field::with_name(
@@ -78,6 +80,7 @@ impl<S: StateStore> LogRowSeqScanExecutor<S> {
             old_epoch,
             new_epoch,
             version_id,
+            ordered,
         }
     }
 }
@@ -146,6 +149,7 @@ impl BoxedExecutorBuilder for LogStoreRowSeqScanExecutorBuilder {
                 chunk_size as usize,
                 source.plan_node().get_identity().clone(),
                 metrics,
+                log_store_seq_scan_node.ordered,
             )))
         })
     }
@@ -175,6 +179,7 @@ impl<S: StateStore> LogRowSeqScanExecutor<S> {
             new_epoch,
             version_id,
             schema,
+            ordered,
             ..
         } = *self;
         let table = std::sync::Arc::new(table);
@@ -194,6 +199,7 @@ impl<S: StateStore> LogRowSeqScanExecutor<S> {
             chunk_size,
             histogram,
             Arc::new(schema.clone()),
+            ordered,
         );
         #[for_await]
         for chunk in stream {
@@ -211,12 +217,14 @@ impl<S: StateStore> LogRowSeqScanExecutor<S> {
         chunk_size: usize,
         histogram: Option<impl Deref<Target = Histogram>>,
         schema: Arc<Schema>,
+        ordered: bool,
     ) {
         // Range Scan.
         let iter = table
             .batch_iter_log_with_pk_bounds(
                 old_epoch,
                 HummockReadEpoch::BatchQueryCommitted(new_epoch, version_id),
+                ordered,
             )
             .await?
             .flat_map(|r| {
diff --git a/src/batch/src/executor/mod.rs b/src/batch/src/executor/mod.rs
index c3bd373198df7..ce84065d9d41c 100644
--- a/src/batch/src/executor/mod.rs
+++ b/src/batch/src/executor/mod.rs
@@ -29,6 +29,7 @@ mod managed;
 mod max_one_row;
 mod merge_sort;
 mod merge_sort_exchange;
+mod mysql_query;
 mod order_by;
 mod postgres_query;
 mod project;
@@ -65,6 +66,7 @@ pub use managed::*;
 pub use max_one_row::*;
 pub use merge_sort::*;
 pub use merge_sort_exchange::*;
+pub use mysql_query::*;
 pub use order_by::*;
 pub use postgres_query::*;
 pub use project::*;
@@ -247,6 +249,7 @@ impl<'a, C: BatchTaskContext> ExecutorBuilder<'a, C> {
             NodeBody::FileScan => FileScanExecutorBuilder,
             NodeBody::IcebergScan => IcebergScanExecutorBuilder,
             NodeBody::PostgresQuery => PostgresQueryExecutorBuilder,
+            NodeBody::MysqlQuery => MySqlQueryExecutorBuilder,
             // Follow NodeBody only used for test
             NodeBody::BlockExecutor => BlockExecutorBuilder,
             NodeBody::BusyLoopExecutor => BusyLoopExecutorBuilder,
diff --git a/src/batch/src/executor/mysql_query.rs b/src/batch/src/executor/mysql_query.rs
new file mode 100644
index 0000000000000..721c9c5e55bf1
--- /dev/null
+++ b/src/batch/src/executor/mysql_query.rs
@@ -0,0 +1,170 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use anyhow::Context;
+use futures_async_stream::try_stream;
+use futures_util::stream::StreamExt;
+use mysql_async;
+use mysql_async::prelude::*;
+use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::row::OwnedRow;
+use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
+use risingwave_connector::parser::mysql_datum_to_rw_datum;
+use risingwave_pb::batch_plan::plan_node::NodeBody;
+
+use crate::error::{BatchError, BatchExternalSystemError};
+use crate::executor::{BoxedExecutor, BoxedExecutorBuilder, DataChunk, Executor, ExecutorBuilder};
+use crate::task::BatchTaskContext;
+
+/// `MySqlQuery` executor. Runs a query against a `MySql` database.
+pub struct MySqlQueryExecutor {
+    schema: Schema,
+    host: String,
+    port: String,
+    username: String,
+    password: String,
+    database: String,
+    query: String,
+    identity: String,
+    chunk_size: usize,
+}
+
+impl Executor for MySqlQueryExecutor {
+    fn schema(&self) -> &risingwave_common::catalog::Schema {
+        &self.schema
+    }
+
+    fn identity(&self) -> &str {
+        &self.identity
+    }
+
+    fn execute(self: Box<Self>) -> super::BoxedDataChunkStream {
+        self.do_execute().boxed()
+    }
+}
+pub fn mysql_row_to_owned_row(
+    mut row: mysql_async::Row,
+    schema: &Schema,
+) -> Result<OwnedRow, BatchError> {
+    let mut datums = vec![];
+    for i in 0..schema.fields.len() {
+        let rw_field = &schema.fields[i];
+        let name = rw_field.name.as_str();
+        let datum = match mysql_datum_to_rw_datum(&mut row, i, name, &rw_field.data_type) {
+            Ok(val) => val,
+            Err(e) => {
+                let e = BatchExternalSystemError(e);
+                return Err(e.into());
+            }
+        };
+        datums.push(datum);
+    }
+    Ok(OwnedRow::new(datums))
+}
+
+impl MySqlQueryExecutor {
+    pub fn new(
+        schema: Schema,
+        host: String,
+        port: String,
+        username: String,
+        password: String,
+        database: String,
+        query: String,
+        identity: String,
+        chunk_size: usize,
+    ) -> Self {
+        Self {
+            schema,
+            host,
+            port,
+            username,
+            password,
+            database,
+            query,
+            identity,
+            chunk_size,
+        }
+    }
+
+    #[try_stream(ok = DataChunk, error = BatchError)]
+    async fn do_execute(self: Box<Self>) {
+        tracing::debug!("mysql_query_executor: started");
+        let database_opts: mysql_async::Opts = mysql_async::OptsBuilder::default()
+            .ip_or_hostname(self.host)
+            .tcp_port(self.port.parse::<u16>().unwrap()) // FIXME
+            .user(Some(self.username))
+            .pass(Some(self.password))
+            .db_name(Some(self.database))
+            .into();
+
+        let pool = mysql_async::Pool::new(database_opts);
+        let mut conn = pool
+            .get_conn()
+            .await
+            .context("failed to connect to mysql in batch executor")?;
+
+        let query = self.query;
+        let mut query_iter = conn
+            .query_iter(query)
+            .await
+            .context("failed to execute my_sql_query in batch executor")?;
+        let Some(row_stream) = query_iter.stream::<mysql_async::Row>().await? else {
+            bail!("failed to get row stream from mysql query")
+        };
+
+        let mut builder = DataChunkBuilder::new(self.schema.data_types(), self.chunk_size);
+        tracing::debug!("mysql_query_executor: query executed, start deserializing rows");
+        // deserialize the rows
+        #[for_await]
+        for row in row_stream {
+            let row = row?;
+            let owned_row = mysql_row_to_owned_row(row, &self.schema)?;
+            if let Some(chunk) = builder.append_one_row(owned_row) {
+                yield chunk;
+            }
+        }
+        if let Some(chunk) = builder.consume_all() {
+            yield chunk;
+        }
+        return Ok(());
+    }
+}
+
+pub struct MySqlQueryExecutorBuilder {}
+
+#[async_trait::async_trait]
+impl BoxedExecutorBuilder for MySqlQueryExecutorBuilder {
+    async fn new_boxed_executor<C: BatchTaskContext>(
+        source: &ExecutorBuilder<'_, C>,
+        _inputs: Vec<BoxedExecutor>,
+    ) -> crate::error::Result<BoxedExecutor> {
+        let mysql_query_node = try_match_expand!(
+            source.plan_node().get_node_body().unwrap(),
+            NodeBody::MysqlQuery
+        )?;
+
+        Ok(Box::new(MySqlQueryExecutor::new(
+            Schema::from_iter(mysql_query_node.columns.iter().map(Field::from)),
+            mysql_query_node.hostname.clone(),
+            mysql_query_node.port.clone(),
+            mysql_query_node.username.clone(),
+            mysql_query_node.password.clone(),
+            mysql_query_node.database.clone(),
+            mysql_query_node.query.clone(),
+            source.plan_node().get_identity().clone(),
+            source.context.get_config().developer.chunk_size,
+        )))
+    }
+}
diff --git a/src/batch/src/executor/postgres_query.rs b/src/batch/src/executor/postgres_query.rs
index 59a9e56d441bc..4ae1fcba65da9 100644
--- a/src/batch/src/executor/postgres_query.rs
+++ b/src/batch/src/executor/postgres_query.rs
@@ -37,6 +37,7 @@ pub struct PostgresQueryExecutor {
     database: String,
     query: String,
     identity: String,
+    chunk_size: usize,
 }
 
 impl Executor for PostgresQueryExecutor {
@@ -115,6 +116,7 @@ impl PostgresQueryExecutor {
         database: String,
         query: String,
         identity: String,
+        chunk_size: usize,
     ) -> Self {
         Self {
             schema,
@@ -125,6 +127,7 @@ impl PostgresQueryExecutor {
             database,
             query,
             identity,
+            chunk_size,
         }
     }
 
@@ -146,15 +149,17 @@ impl PostgresQueryExecutor {
             }
         });
 
-        // TODO(kwannoel): Use pagination using CURSOR.
-        let rows = client
-            .query(&self.query, &[])
+        let params: &[&str] = &[];
+        let row_stream = client
+            .query_raw(&self.query, params)
             .await
             .context("postgres_query received error from remote server")?;
-        let mut builder = DataChunkBuilder::new(self.schema.data_types(), 1024);
+        let mut builder = DataChunkBuilder::new(self.schema.data_types(), self.chunk_size);
         tracing::debug!("postgres_query_executor: query executed, start deserializing rows");
         // deserialize the rows
-        for row in rows {
+        #[for_await]
+        for row in row_stream {
+            let row = row?;
             let owned_row = postgres_row_to_owned_row(row, &self.schema)?;
             if let Some(chunk) = builder.append_one_row(owned_row) {
                 yield chunk;
@@ -189,6 +194,7 @@ impl BoxedExecutorBuilder for PostgresQueryExecutorBuilder {
             postgres_query_node.database.clone(),
             postgres_query_node.query.clone(),
             source.plan_node().get_identity().clone(),
+            source.context.get_config().developer.chunk_size,
         )))
     }
 }
diff --git a/src/batch/src/executor/s3_file_scan.rs b/src/batch/src/executor/s3_file_scan.rs
index 5096d1d625fa1..a7b0d1bacd791 100644
--- a/src/batch/src/executor/s3_file_scan.rs
+++ b/src/batch/src/executor/s3_file_scan.rs
@@ -109,7 +109,7 @@ impl S3FileScanExecutor {
 
             #[for_await]
             for record_batch in record_batch_stream {
-                let record_batch = record_batch.map_err(BatchError::Parquet)?;
+                let record_batch = record_batch?;
                 let chunk = IcebergArrowConvert.chunk_from_record_batch(&record_batch)?;
                 debug_assert_eq!(chunk.data_types(), self.schema.data_types());
                 yield chunk;
diff --git a/src/batch/src/lib.rs b/src/batch/src/lib.rs
index 9b88c3be9cd68..38eb0efd80553 100644
--- a/src/batch/src/lib.rs
+++ b/src/batch/src/lib.rs
@@ -20,7 +20,6 @@
 #![feature(coroutines)]
 #![feature(proc_macro_hygiene, stmt_expr_attributes)]
 #![feature(iterator_try_collect)]
-#![feature(is_sorted)]
 #![recursion_limit = "256"]
 #![feature(let_chains)]
 #![feature(int_roundings)]
diff --git a/src/batch/src/worker_manager/worker_node_manager.rs b/src/batch/src/worker_manager/worker_node_manager.rs
index 772bc8a4b6da7..98923bac62ba5 100644
--- a/src/batch/src/worker_manager/worker_node_manager.rs
+++ b/src/batch/src/worker_manager/worker_node_manager.rs
@@ -222,10 +222,7 @@ impl WorkerNodeManager {
                 // Do nothing for placeholder fragment.
             }
             None => {
-                panic!(
-                    "Streaming vnode mapping not found for fragment_id: {}",
-                    fragment_id
-                )
+                tracing::warn!(fragment_id, "Streaming vnode mapping not found");
             }
         };
     }
diff --git a/src/bench/s3_bench/main.rs b/src/bench/s3_bench/main.rs
index 792c9c4743dbf..6720a492ba942 100644
--- a/src/bench/s3_bench/main.rs
+++ b/src/bench/s3_bench/main.rs
@@ -557,7 +557,7 @@ where
 {
     struct StringVisitor;
 
-    impl<'de> serde::de::Visitor<'de> for StringVisitor {
+    impl serde::de::Visitor<'_> for StringVisitor {
         type Value = ByteSize;
 
         fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
diff --git a/src/bench/sink_bench/main.rs b/src/bench/sink_bench/main.rs
index 05eabdb96e913..850c41c31460f 100644
--- a/src/bench/sink_bench/main.rs
+++ b/src/bench/sink_bench/main.rs
@@ -383,10 +383,10 @@ where
         sink_writer_param.vnode_bitmap = Some(Bitmap::ones(1));
     }
     let log_sinker = sink.new_log_sinker(sink_writer_param).await.unwrap();
-    if let Err(e) = log_sinker.consume_log_and_sink(&mut log_reader).await {
-        return Err(e.to_report_string());
+    match log_sinker.consume_log_and_sink(&mut log_reader).await {
+        Ok(_) => Err("Stream closed".to_string()),
+        Err(e) => Err(e.to_report_string()),
     }
-    Err("Stream closed".to_string())
 }
 
 #[derive(Debug, Deserialize)]
diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml
index e84a6675f04b4..4b002bd8084c6 100644
--- a/src/common/Cargo.toml
+++ b/src/common/Cargo.toml
@@ -97,7 +97,7 @@ serde_default = "0.2"
 serde_json = "1"
 serde_with = "3"
 smallbitset = "0.7.1"
-speedate = "0.14.0"
+speedate = "0.15.0"
 stacker = "0.1"
 static_assertions = "1"
 strum = "0.26"
@@ -128,8 +128,8 @@ uuid = { version = "1", features = ["v4"] }
 [target.'cfg(not(madsim))'.dependencies]
 workspace-hack = { path = "../workspace-hack" }
 http-body = "0.4.5"
-tower-layer = "0.3.2"
-tower-service = "0.3.2"
+tower-layer = "0.3.3"
+tower-service = "0.3.3"
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs = { version = "0.17", default-features = false }
diff --git a/src/common/common_service/Cargo.toml b/src/common/common_service/Cargo.toml
index 23d1db39af077..37775ff04a82b 100644
--- a/src/common/common_service/Cargo.toml
+++ b/src/common/common_service/Cargo.toml
@@ -29,8 +29,8 @@ thiserror = "1"
 thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio", features = ["rt", "rt-multi-thread", "sync", "macros", "time", "signal"] }
 tonic = { workspace = true }
-tower = { version = "0.4", features = ["util", "load-shed"] }
-tower-http = { version = "0.5", features = ["add-extension", "compression-gzip"] }
+tower = { version = "0.5", features = ["util", "load-shed"] }
+tower-http = { version = "0.6", features = ["add-extension", "compression-gzip"] }
 tracing = "0.1"
 
 [target.'cfg(not(madsim))'.dependencies]
diff --git a/src/common/estimate_size/src/collections/btreemap.rs b/src/common/estimate_size/src/collections/btreemap.rs
index af9ab3471acec..25d0c5f82c677 100644
--- a/src/common/estimate_size/src/collections/btreemap.rs
+++ b/src/common/estimate_size/src/collections/btreemap.rs
@@ -224,7 +224,7 @@ pub struct OccupiedEntry<'a, K, V> {
     heap_size: &'a mut KvSize,
 }
 
-impl<'a, K, V> OccupiedEntry<'a, K, V>
+impl<K, V> OccupiedEntry<'_, K, V>
 where
     K: EstimateSize + Ord,
     V: EstimateSize,
@@ -248,7 +248,7 @@ where
     heap_size: &'a mut KvSize,
 }
 
-impl<'a, K, V, F> Iterator for ExtractIf<'a, K, V, F>
+impl<K, V, F> Iterator for ExtractIf<'_, K, V, F>
 where
     K: EstimateSize,
     V: EstimateSize,
diff --git a/src/common/estimate_size/src/collections/mod.rs b/src/common/estimate_size/src/collections/mod.rs
index 5bffd5133eddc..c5c8a03d71761 100644
--- a/src/common/estimate_size/src/collections/mod.rs
+++ b/src/common/estimate_size/src/collections/mod.rs
@@ -85,7 +85,7 @@ where
     }
 }
 
-impl<'a, V, S> Drop for MutGuard<'a, V, S>
+impl<V, S> Drop for MutGuard<'_, V, S>
 where
     V: EstimateSize,
     S: private::GenericKvSize,
@@ -96,7 +96,7 @@ where
     }
 }
 
-impl<'a, V, S> Deref for MutGuard<'a, V, S>
+impl<V, S> Deref for MutGuard<'_, V, S>
 where
     V: EstimateSize,
     S: private::GenericKvSize,
@@ -108,7 +108,7 @@ where
     }
 }
 
-impl<'a, V, S> DerefMut for MutGuard<'a, V, S>
+impl<V, S> DerefMut for MutGuard<'_, V, S>
 where
     V: EstimateSize,
     S: private::GenericKvSize,
diff --git a/src/common/metrics/Cargo.toml b/src/common/metrics/Cargo.toml
index c4c20bd704310..78562624c6935 100644
--- a/src/common/metrics/Cargo.toml
+++ b/src/common/metrics/Cargo.toml
@@ -37,8 +37,8 @@ serde = { version = "1", features = ["derive"] }
 thiserror-ext = { workspace = true }
 tokio = { version = "0.2", package = "madsim-tokio" }
 tonic = { workspace = true }
-tower-layer = "0.3.2"
-tower-service = "0.3.2"
+tower-layer = "0.3.3"
+tower-service = "0.3.3"
 tracing = "0.1"
 tracing-subscriber = "0.3.17"
 
diff --git a/src/common/metrics/src/gauge_ext.rs b/src/common/metrics/src/gauge_ext.rs
index d357ae5e52cb3..b76f2ac7531d0 100644
--- a/src/common/metrics/src/gauge_ext.rs
+++ b/src/common/metrics/src/gauge_ext.rs
@@ -30,7 +30,7 @@ impl IntGauge {
             }
         }
 
-        impl<'a> Drop for Guard<'a> {
+        impl Drop for Guard<'_> {
             fn drop(&mut self) {
                 self.gauge.dec();
             }
diff --git a/src/common/metrics/src/guarded_metrics.rs b/src/common/metrics/src/guarded_metrics.rs
index 9b16cc778938c..97ea311455270 100644
--- a/src/common/metrics/src/guarded_metrics.rs
+++ b/src/common/metrics/src/guarded_metrics.rs
@@ -191,6 +191,8 @@ impl<const N: usize> LabelGuardedMetricsInfo<N> {
     }
 }
 
+/// An RAII metrics vec with labels.
+///
 /// `LabelGuardedMetricVec` enhances the [`MetricVec`] to ensure the set of labels to be
 /// correctly removed from the Prometheus client once being dropped. This is useful for metrics
 /// that are associated with an object that can be dropped, such as streaming jobs, fragments,
diff --git a/src/common/proc_macro/src/lib.rs b/src/common/proc_macro/src/lib.rs
index ccf7d4c282e2b..52761b5891109 100644
--- a/src/common/proc_macro/src/lib.rs
+++ b/src/common/proc_macro/src/lib.rs
@@ -268,6 +268,7 @@ pub fn session_config(input: TokenStream) -> TokenStream {
 
 /// This proc macro recursively extracts rustdoc comments from the fields in a struct and generates a method
 /// that produces docs for each field.
+///
 /// Unlike rustdoc, this tool focuses solely on extracting rustdoc for struct fields, without methods.
 ///
 /// Example:
diff --git a/src/common/secret/Cargo.toml b/src/common/secret/Cargo.toml
index 4b698a737cb54..6d501594acbde 100644
--- a/src/common/secret/Cargo.toml
+++ b/src/common/secret/Cargo.toml
@@ -25,5 +25,8 @@ thiserror = "1"
 thiserror-ext = { workspace = true }
 tracing = "0.1"
 
+[dev-dependencies]
+hex = "0.4"
+
 [lints]
 workspace = true
diff --git a/src/common/secret/src/encryption.rs b/src/common/secret/src/encryption.rs
index a6c0253fb1f9c..c6d5e7cdb66de 100644
--- a/src/common/secret/src/encryption.rs
+++ b/src/common/secret/src/encryption.rs
@@ -71,7 +71,8 @@ mod test {
     use super::*;
     #[test]
     fn test_secret_encryption_decyption() {
-        let key = b"0123456789abcdef";
+        let key = &hex::decode("0123456789abcdef0123456789abcdef").unwrap();
+        assert!(key.len() == 16);
         let plaintext = "Hello, world!".as_bytes();
         let secret = SecretEncryption::encrypt(key, plaintext).unwrap();
         let decrypted = secret.decrypt(key).unwrap();
diff --git a/src/common/src/acl/mod.rs b/src/common/src/acl/mod.rs
index b1c41bc8ec6ad..b0bf939a2353c 100644
--- a/src/common/src/acl/mod.rs
+++ b/src/common/src/acl/mod.rs
@@ -104,7 +104,7 @@ pub static ALL_AVAILABLE_TABLE_MODES: LazyLock<AclModeSet> =
     LazyLock::new(|| make_bitflags!(AclMode::{Select | Insert | Update | Delete}).into());
 pub static ALL_AVAILABLE_SOURCE_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::readonly);
 pub static ALL_AVAILABLE_MVIEW_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::readonly);
-pub static ALL_AVAILABLE_SINK_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::empty);
+pub static ALL_AVAILABLE_SINK_MODES: LazyLock<AclModeSet> = LazyLock::new(AclModeSet::readonly);
 pub static ALL_AVAILABLE_SUBSCRIPTION_MODES: LazyLock<AclModeSet> =
     LazyLock::new(AclModeSet::empty);
 pub static ALL_AVAILABLE_FUNCTION_MODES: LazyLock<AclModeSet> =
diff --git a/src/common/src/array/arrow/arrow_iceberg.rs b/src/common/src/array/arrow/arrow_iceberg.rs
index 9cb7df6f809a7..e353b57e257c9 100644
--- a/src/common/src/array/arrow/arrow_iceberg.rs
+++ b/src/common/src/array/arrow/arrow_iceberg.rs
@@ -83,8 +83,8 @@ impl IcebergArrowConvert {
 impl ToArrow for IcebergArrowConvert {
     #[inline]
     fn decimal_type_to_arrow(&self, name: &str) -> arrow_schema::Field {
-        let data_type =
-            arrow_schema::DataType::Decimal128(arrow_schema::DECIMAL128_MAX_PRECISION, 0);
+        // Fixed-point decimal; precision P, scale S Scale is fixed, precision must be less than 38.
+        let data_type = arrow_schema::DataType::Decimal128(28, 10);
         arrow_schema::Field::new(name, data_type, true)
     }
 
@@ -138,8 +138,9 @@ impl ToArrow for IcebergArrowConvert {
 impl FromArrow for IcebergArrowConvert {}
 
 /// Iceberg sink with `create_table_if_not_exists` option will use this struct to convert the
-/// iceberg data type to arrow data type. Specifically, it will add the field id to the
-/// arrow field metadata, because iceberg-rust and icelake need the field id to be set.
+/// iceberg data type to arrow data type.
+///
+/// Specifically, it will add the field id to the arrow field metadata, because iceberg-rust and icelake need the field id to be set.
 ///
 /// Note: this is different from [`IcebergArrowConvert`], which is used to read from/write to
 /// an _existing_ iceberg table. In that case, we just need to make sure the data is compatible to the existing schema.
@@ -178,7 +179,7 @@ impl ToArrow for IcebergCreateTableArrowConvert {
         // We choose 28 here
         // The decimal type finally will be converted to an iceberg decimal type.
         // Iceberg decimal(P,S)
-        // Fixed-point decimal; precision P, scale S Scale is fixed, precision must be 38 or less.
+        // Fixed-point decimal; precision P, scale S Scale is fixed, precision must be less than 38.
         let data_type = arrow_schema::DataType::Decimal128(28, 10);
 
         let mut arrow_field = arrow_schema::Field::new(name, data_type, true);
diff --git a/src/common/src/array/bytes_array.rs b/src/common/src/array/bytes_array.rs
index 0d62c3d26c6e3..2b19451822ad0 100644
--- a/src/common/src/array/bytes_array.rs
+++ b/src/common/src/array/bytes_array.rs
@@ -271,7 +271,7 @@ pub struct PartialBytesWriter<'a> {
     builder: &'a mut BytesArrayBuilder,
 }
 
-impl<'a> PartialBytesWriter<'a> {
+impl PartialBytesWriter<'_> {
     /// `write_ref` will append partial dirty data to `builder`.
     /// `PartialBytesWriter::write_ref` is different from `BytesWriter::write_ref`
     /// in that it allows us to call it multiple times.
@@ -287,7 +287,7 @@ impl<'a> PartialBytesWriter<'a> {
     }
 }
 
-impl<'a> Drop for PartialBytesWriter<'a> {
+impl Drop for PartialBytesWriter<'_> {
     fn drop(&mut self) {
         // If `finish` is not called, we should rollback the data.
         self.builder.rollback_partial();
diff --git a/src/common/src/array/data_chunk_iter.rs b/src/common/src/array/data_chunk_iter.rs
index 064de8532c690..24d6271b547a8 100644
--- a/src/common/src/array/data_chunk_iter.rs
+++ b/src/common/src/array/data_chunk_iter.rs
@@ -85,7 +85,7 @@ impl<'a> Iterator for DataChunkRefIter<'a> {
     }
 }
 
-impl<'a> FusedIterator for DataChunkRefIter<'a> {}
+impl FusedIterator for DataChunkRefIter<'_> {}
 
 pub struct DataChunkRefIterWithHoles<'a> {
     chunk: &'a DataChunk,
@@ -132,7 +132,7 @@ mod row_ref {
         idx: usize,
     }
 
-    impl<'a> std::fmt::Debug for RowRef<'a> {
+    impl std::fmt::Debug for RowRef<'_> {
         fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
             f.debug_list().entries(self.iter()).finish()
         }
diff --git a/src/common/src/array/iterator.rs b/src/common/src/array/iterator.rs
index 31518150e6f8d..fa39f1ab2e63a 100644
--- a/src/common/src/array/iterator.rs
+++ b/src/common/src/array/iterator.rs
@@ -47,8 +47,8 @@ impl<'a, A: Array> Iterator for ArrayIterator<'a, A> {
     }
 }
 
-impl<'a, A: Array> ExactSizeIterator for ArrayIterator<'a, A> {}
-unsafe impl<'a, A: Array> TrustedLen for ArrayIterator<'a, A> {}
+impl<A: Array> ExactSizeIterator for ArrayIterator<'_, A> {}
+unsafe impl<A: Array> TrustedLen for ArrayIterator<'_, A> {}
 
 #[cfg(test)]
 mod tests {
diff --git a/src/common/src/array/mod.rs b/src/common/src/array/mod.rs
index b34e5f9b9c470..e8d4cad8fb5b5 100644
--- a/src/common/src/array/mod.rs
+++ b/src/common/src/array/mod.rs
@@ -731,10 +731,10 @@ mod test_util {
     use crate::util::iter_util::ZipEqFast;
 
     pub fn hash_finish<H: Hasher>(hashers: &[H]) -> Vec<u64> {
-        return hashers
+        hashers
             .iter()
             .map(|hasher| hasher.finish())
-            .collect::<Vec<u64>>();
+            .collect::<Vec<u64>>()
     }
 
     pub fn test_hash<H: BuildHasher, A: Array>(arrs: Vec<A>, expects: Vec<u64>, hasher_builder: H) {
diff --git a/src/common/src/array/stream_record.rs b/src/common/src/array/stream_record.rs
index 1037f92b286ce..cfd474017d933 100644
--- a/src/common/src/array/stream_record.rs
+++ b/src/common/src/array/stream_record.rs
@@ -28,7 +28,7 @@ pub enum RecordType {
 }
 
 /// Generic type to represent a row change.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Copy)]
 pub enum Record<R: Row> {
     Insert { new_row: R },
     Delete { old_row: R },
diff --git a/src/common/src/array/utf8_array.rs b/src/common/src/array/utf8_array.rs
index 8463f73e3ee58..749a9efe3da13 100644
--- a/src/common/src/array/utf8_array.rs
+++ b/src/common/src/array/utf8_array.rs
@@ -204,7 +204,7 @@ pub struct PartialStringWriter<'a> {
     bytes: PartialBytesWriter<'a>,
 }
 
-impl<'a> PartialStringWriter<'a> {
+impl PartialStringWriter<'_> {
     /// `finish` will be called while the entire record is written.
     /// Exactly one new record was appended and the `builder` can be safely used.
     pub fn finish(self) {
diff --git a/src/common/src/bitmap.rs b/src/common/src/bitmap.rs
index ae07105164408..995771970ece3 100644
--- a/src/common/src/bitmap.rs
+++ b/src/common/src/bitmap.rs
@@ -42,6 +42,7 @@ use std::ops::{BitAnd, BitAndAssign, BitOr, BitOrAssign, BitXor, Not, Range, Ran
 use risingwave_common_estimate_size::EstimateSize;
 use risingwave_pb::common::buffer::CompressionType;
 use risingwave_pb::common::PbBuffer;
+use rw_iter_util::ZipEqFast;
 
 #[derive(Default, Debug, Clone, EstimateSize)]
 pub struct BitmapBuilder {
@@ -325,7 +326,7 @@ impl Bitmap {
 
     /// Returns the length of vector to store `num_bits` bits.
     fn vec_len(num_bits: usize) -> usize {
-        (num_bits + BITS - 1) / BITS
+        num_bits.div_ceil(BITS)
     }
 
     /// Returns the number of valid bits in the bitmap,
@@ -468,7 +469,7 @@ impl From<usize> for Bitmap {
     }
 }
 
-impl<'a, 'b> BitAnd<&'b Bitmap> for &'a Bitmap {
+impl<'b> BitAnd<&'b Bitmap> for &Bitmap {
     type Output = Bitmap;
 
     fn bitand(self, rhs: &'b Bitmap) -> Bitmap {
@@ -488,7 +489,7 @@ impl<'a, 'b> BitAnd<&'b Bitmap> for &'a Bitmap {
     }
 }
 
-impl<'a> BitAnd<Bitmap> for &'a Bitmap {
+impl BitAnd<Bitmap> for &Bitmap {
     type Output = Bitmap;
 
     fn bitand(self, rhs: Bitmap) -> Self::Output {
@@ -524,7 +525,7 @@ impl BitAndAssign<Bitmap> for Bitmap {
     }
 }
 
-impl<'a, 'b> BitOr<&'b Bitmap> for &'a Bitmap {
+impl<'b> BitOr<&'b Bitmap> for &Bitmap {
     type Output = Bitmap;
 
     fn bitor(self, rhs: &'b Bitmap) -> Bitmap {
@@ -544,7 +545,7 @@ impl<'a, 'b> BitOr<&'b Bitmap> for &'a Bitmap {
     }
 }
 
-impl<'a> BitOr<Bitmap> for &'a Bitmap {
+impl BitOr<Bitmap> for &Bitmap {
     type Output = Bitmap;
 
     fn bitor(self, rhs: Bitmap) -> Self::Output {
@@ -599,7 +600,7 @@ impl BitXor for &Bitmap {
     }
 }
 
-impl<'a> Not for &'a Bitmap {
+impl Not for &Bitmap {
     type Output = Bitmap;
 
     fn not(self) -> Self::Output {
@@ -700,7 +701,7 @@ pub struct BitmapIter<'a> {
     all_ones: bool,
 }
 
-impl<'a> BitmapIter<'a> {
+impl BitmapIter<'_> {
     fn next_always_load_usize(&mut self) -> Option<bool> {
         if self.idx >= self.num_bits {
             return None;
@@ -724,7 +725,7 @@ impl<'a> BitmapIter<'a> {
     }
 }
 
-impl<'a> iter::Iterator for BitmapIter<'a> {
+impl iter::Iterator for BitmapIter<'_> {
     type Item = bool;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -778,7 +779,7 @@ pub enum BitmapOnesIter<'a> {
     },
 }
 
-impl<'a> iter::Iterator for BitmapOnesIter<'a> {
+impl iter::Iterator for BitmapOnesIter<'_> {
     type Item = usize;
 
     fn next(&mut self) -> Option<Self::Item> {
@@ -826,6 +827,15 @@ impl<'a> iter::Iterator for BitmapOnesIter<'a> {
     }
 }
 
+pub trait FilterByBitmap: ExactSizeIterator + Sized {
+    fn filter_by_bitmap(self, bitmap: &Bitmap) -> impl Iterator<Item = Self::Item> {
+        self.zip_eq_fast(bitmap.iter())
+            .filter_map(|(item, bit)| bit.then_some(item))
+    }
+}
+
+impl<T> FilterByBitmap for T where T: ExactSizeIterator {}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/common/src/cache.rs b/src/common/src/cache.rs
index e86ef432eea95..678a101ae62a6 100644
--- a/src/common/src/cache.rs
+++ b/src/common/src/cache.rs
@@ -877,13 +877,13 @@ pub struct CleanCacheGuard<'a, K: LruKey + Clone + 'static, T: LruValue + 'stati
     hash: u64,
 }
 
-impl<'a, K: LruKey + Clone + 'static, T: LruValue + 'static> CleanCacheGuard<'a, K, T> {
+impl<K: LruKey + Clone + 'static, T: LruValue + 'static> CleanCacheGuard<'_, K, T> {
     fn mark_success(mut self) -> K {
         self.key.take().unwrap()
     }
 }
 
-impl<'a, K: LruKey + Clone + 'static, T: LruValue + 'static> Drop for CleanCacheGuard<'a, K, T> {
+impl<K: LruKey + Clone + 'static, T: LruValue + 'static> Drop for CleanCacheGuard<'_, K, T> {
     fn drop(&mut self) {
         if let Some(key) = self.key.as_ref() {
             self.cache.clear_pending_request(key, self.hash);
diff --git a/src/common/src/catalog/mod.rs b/src/common/src/catalog/mod.rs
index 86c6e8895c066..23cdf04cdf8a9 100644
--- a/src/common/src/catalog/mod.rs
+++ b/src/common/src/catalog/mod.rs
@@ -157,7 +157,7 @@ pub trait SysCatalogReader: Sync + Send + 'static {
 
 pub type SysCatalogReaderRef = Arc<dyn SysCatalogReader>;
 
-#[derive(Clone, Debug, Default, Display, Hash, PartialOrd, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, Display, Hash, PartialOrd, PartialEq, Eq, Copy)]
 #[display("{database_id}")]
 pub struct DatabaseId {
     pub database_id: u32,
diff --git a/src/common/src/catalog/physical_table.rs b/src/common/src/catalog/physical_table.rs
index df1b30fd41eeb..680f23fb0dc14 100644
--- a/src/common/src/catalog/physical_table.rs
+++ b/src/common/src/catalog/physical_table.rs
@@ -21,7 +21,7 @@ use risingwave_pb::plan_common::StorageTableDesc;
 
 use super::{ColumnDesc, ColumnId, TableId};
 use crate::catalog::get_dist_key_in_pk_indices;
-use crate::hash::VnodeCountCompat;
+use crate::hash::{VnodeCount, VnodeCountCompat};
 use crate::util::sort_util::ColumnOrder;
 
 /// Includes necessary information for compute node to access data of the table.
@@ -117,7 +117,7 @@ impl TableDesc {
             versioned: self.versioned,
             stream_key: self.stream_key.iter().map(|&x| x as u32).collect(),
             vnode_col_idx_in_pk,
-            maybe_vnode_count: Some(self.vnode_count as u32),
+            maybe_vnode_count: VnodeCount::set(self.vnode_count).to_protobuf(),
         })
     }
 
diff --git a/src/common/src/config.rs b/src/common/src/config.rs
index 5f7bb2bdeff36..c3fc78919a3ef 100644
--- a/src/common/src/config.rs
+++ b/src/common/src/config.rs
@@ -116,7 +116,7 @@ pub trait OverrideConfig {
     fn r#override(&self, config: &mut RwConfig);
 }
 
-impl<'a, T: OverrideConfig> OverrideConfig for &'a T {
+impl<T: OverrideConfig> OverrideConfig for &T {
     fn r#override(&self, config: &mut RwConfig) {
         T::r#override(self, config)
     }
@@ -195,6 +195,10 @@ pub struct MetaConfig {
     #[serde(default = "default::meta::full_gc_object_limit")]
     pub full_gc_object_limit: u64,
 
+    /// Duration in seconds to retain garbage collection history data.
+    #[serde(default = "default::meta::gc_history_retention_time_sec")]
+    pub gc_history_retention_time_sec: u64,
+
     /// Max number of inflight time travel query.
     #[serde(default = "default::meta::max_inflight_time_travel_query")]
     pub max_inflight_time_travel_query: u64,
@@ -301,16 +305,12 @@ pub struct MetaConfig {
     #[serde(default = "default::meta::periodic_tombstone_reclaim_compaction_interval_sec")]
     pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
 
-    #[serde(
-        default = "default::meta::periodic_scheduling_compaction_group_interval_sec",
-        alias = "periodic_split_compact_group_interval_sec"
-    )]
-    pub periodic_scheduling_compaction_group_interval_sec: u64,
-
     #[serde(default = "default::meta::move_table_size_limit")]
+    #[deprecated]
     pub move_table_size_limit: u64,
 
     #[serde(default = "default::meta::split_group_size_limit")]
+    #[deprecated]
     pub split_group_size_limit: u64,
 
     #[serde(default = "default::meta::cut_table_size_limit")]
@@ -318,6 +318,7 @@ pub struct MetaConfig {
     pub cut_table_size_limit: u64,
 
     #[serde(default, flatten)]
+    #[config_doc(omitted)]
     pub unrecognized: Unrecognized<Self>,
 
     /// Whether config object storage bucket lifecycle to purge stale data.
@@ -329,14 +330,19 @@ pub struct MetaConfig {
     #[serde(default = "default::meta::partition_vnode_count")]
     pub partition_vnode_count: u32,
 
-    /// The threshold of write throughput to trigger a group split. Increase this configuration value to avoid split too many groups with few data write.
-    #[serde(default = "default::meta::table_write_throughput_threshold")]
-    pub table_write_throughput_threshold: u64,
+    /// The threshold of write throughput to trigger a group split.
+    #[serde(
+        default = "default::meta::table_high_write_throughput_threshold",
+        alias = "table_write_throughput_threshold"
+    )]
+    pub table_high_write_throughput_threshold: u64,
 
-    #[serde(default = "default::meta::min_table_split_write_throughput")]
-    /// If the size of one table is smaller than `min_table_split_write_throughput`, we would not
-    /// split it to an single group.
-    pub min_table_split_write_throughput: u64,
+    #[serde(
+        default = "default::meta::table_low_write_throughput_threshold",
+        alias = "min_table_split_write_throughput"
+    )]
+    /// The threshold of write throughput to trigger a group merge.
+    pub table_low_write_throughput_threshold: u64,
 
     // If the compaction task does not report heartbeat beyond the
     // `compaction_task_max_heartbeat_interval_secs` interval, we will cancel the task
@@ -359,16 +365,6 @@ pub struct MetaConfig {
     #[serde(default = "default::meta::hybrid_partition_vnode_count")]
     pub hybrid_partition_vnode_count: u32,
 
-    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
-    /// Set it max value of 64-bit number to disable this feature.
-    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
-    pub compact_task_table_size_partition_threshold_low: u64,
-
-    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
-    /// Set it max value of 64-bit number to disable this feature.
-    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
-    pub compact_task_table_size_partition_threshold_high: u64,
-
     #[serde(default = "default::meta::event_log_enabled")]
     pub event_log_enabled: bool,
     /// Keeps the latest N events per channel.
@@ -381,6 +377,54 @@ pub struct MetaConfig {
     /// Whether compactor should rewrite row to remove dropped column.
     #[serde(default = "default::meta::enable_dropped_column_reclaim")]
     pub enable_dropped_column_reclaim: bool,
+
+    /// Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`.
+    #[serde(default = "default::meta::split_group_size_ratio")]
+    pub split_group_size_ratio: f64,
+
+    // During group scheduling, the configured `*_throughput_ratio` is used to determine if the sample exceeds the threshold.
+    // Use `table_stat_throuput_window_seconds_for_*` to check if the split and merge conditions are met.
+    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
+    #[serde(default = "default::meta::table_stat_high_write_throughput_ratio_for_split")]
+    pub table_stat_high_write_throughput_ratio_for_split: f64,
+
+    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
+    #[serde(default = "default::meta::table_stat_low_write_throughput_ratio_for_merge")]
+    pub table_stat_low_write_throughput_ratio_for_merge: f64,
+
+    // Hummock also control the size of samples to be judged during group scheduling by `table_stat_sample_size_for_split` and `table_stat_sample_size_for_merge`.
+    // Will use max(table_stat_throuput_window_seconds_for_split /ckpt, table_stat_throuput_window_seconds_for_merge/ckpt) as the global sample size.
+    // For example, if `table_stat_throuput_window_seconds_for_merge` = 240 and `table_stat_throuput_window_seconds_for_split` = 60, and `ckpt_sec = 1`,
+    //  global sample size will be max(240/1, 60/1), then only the last 60 samples will be considered for split, and so on.
+    /// The window seconds of table throughput statistic history for split compaction group.
+    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_split")]
+    pub table_stat_throuput_window_seconds_for_split: usize,
+
+    /// The window seconds of table throughput statistic history for merge compaction group.
+    #[serde(default = "default::meta::table_stat_throuput_window_seconds_for_merge")]
+    pub table_stat_throuput_window_seconds_for_merge: usize,
+
+    /// The threshold of table size in one compact task to decide whether to partition one table into `hybrid_partition_vnode_count` parts, which belongs to default group and materialized view group.
+    /// Set it max value of 64-bit number to disable this feature.
+    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_low")]
+    pub compact_task_table_size_partition_threshold_low: u64,
+
+    /// The threshold of table size in one compact task to decide whether to partition one table into `partition_vnode_count` parts, which belongs to default group and materialized view group.
+    /// Set it max value of 64-bit number to disable this feature.
+    #[serde(default = "default::meta::compact_task_table_size_partition_threshold_high")]
+    pub compact_task_table_size_partition_threshold_high: u64,
+
+    #[serde(
+        default = "default::meta::periodic_scheduling_compaction_group_split_interval_sec",
+        alias = "periodic_split_compact_group_interval_sec"
+    )]
+    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
+
+    #[serde(default = "default::meta::periodic_scheduling_compaction_group_merge_interval_sec")]
+    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
+    #[serde(default)]
+    #[config_doc(nested)]
+    pub meta_store_config: MetaStoreConfig,
 }
 
 #[derive(Copy, Clone, Debug, Default)]
@@ -608,22 +652,36 @@ pub use risingwave_common_metrics::MetricLevel;
 /// the section `[storage.cache]` in `risingwave.toml`.
 #[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
 pub struct CacheConfig {
+    /// Configure the capacity of the block cache in MB explicitly.
+    /// The overridden value will only be effective if:
+    /// 1. `meta_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
+    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
     #[serde(default)]
     pub block_cache_capacity_mb: Option<usize>,
 
+    /// Configure the number of shards in the block cache explicitly.
+    /// If not set, the shard number will be determined automatically based on cache capacity.
     #[serde(default)]
     pub block_cache_shard_num: Option<usize>,
 
     #[serde(default)]
+    #[config_doc(omitted)]
     pub block_cache_eviction: CacheEvictionConfig,
 
+    /// Configure the capacity of the block cache in MB explicitly.
+    /// The overridden value will only be effective if:
+    /// 1. `block_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly.
+    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
     #[serde(default)]
     pub meta_cache_capacity_mb: Option<usize>,
 
+    /// Configure the number of shards in the meta cache explicitly.
+    /// If not set, the shard number will be determined automatically based on cache capacity.
     #[serde(default)]
     pub meta_cache_shard_num: Option<usize>,
 
     #[serde(default)]
+    #[config_doc(omitted)]
     pub meta_cache_eviction: CacheEvictionConfig,
 }
 
@@ -667,8 +725,10 @@ pub struct StorageConfig {
     #[serde(default = "default::storage::share_buffer_compaction_worker_threads_number")]
     pub share_buffer_compaction_worker_threads_number: u32,
 
-    /// Maximum shared buffer size, writes attempting to exceed the capacity will stall until there
-    /// is enough space.
+    /// Configure the maximum shared buffer size in MB explicitly. Writes attempting to exceed the capacity
+    /// will stall until there is enough space. The overridden value will only be effective if:
+    /// 1. `block_cache_capacity_mb` and `meta_cache_capacity_mb` are also configured explicitly.
+    /// 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory.
     #[serde(default)]
     pub shared_buffer_capacity_mb: Option<usize>,
 
@@ -692,6 +752,7 @@ pub struct StorageConfig {
     pub write_conflict_detection_enabled: bool,
 
     #[serde(default)]
+    #[config_doc(nested)]
     pub cache: CacheConfig,
 
     /// DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_capacity_mb` instead.
@@ -746,12 +807,15 @@ pub struct StorageConfig {
     pub min_sstable_size_mb: u32,
 
     #[serde(default)]
+    #[config_doc(nested)]
     pub data_file_cache: FileCacheConfig,
 
     #[serde(default)]
+    #[config_doc(nested)]
     pub meta_file_cache: FileCacheConfig,
 
     #[serde(default)]
+    #[config_doc(nested)]
     pub cache_refill: CacheRefillConfig,
 
     /// Whether to enable streaming upload for sstable.
@@ -787,8 +851,9 @@ pub struct StorageConfig {
     #[serde(default = "default::storage::compactor_iter_max_io_retry_times")]
     pub compactor_iter_max_io_retry_times: usize,
 
-    /// The window size of table info statistic history.
+    /// Deprecated: The window size of table info statistic history.
     #[serde(default = "default::storage::table_info_statistic_history_times")]
+    #[deprecated]
     pub table_info_statistic_history_times: usize,
 
     #[serde(default, flatten)]
@@ -1039,8 +1104,7 @@ pub struct StreamingDeveloperConfig {
     /// Enable arrangement backfill
     /// If false, the arrangement backfill will be disabled,
     /// even if session variable set.
-    /// If true, it will be enabled by default, but session variable
-    /// can override it.
+    /// If true, it's decided by session variable `streaming_use_arrangement_backfill` (default true)
     pub enable_arrangement_backfill: bool,
 
     #[serde(default = "default::developer::stream_high_join_amplification_threshold")]
@@ -1060,6 +1124,13 @@ pub struct StreamingDeveloperConfig {
     /// A flag to allow disabling the auto schema change handling
     #[serde(default = "default::developer::stream_enable_auto_schema_change")]
     pub enable_auto_schema_change: bool,
+
+    #[serde(default = "default::developer::enable_shared_source")]
+    /// Enable shared source
+    /// If false, the shared source will be disabled,
+    /// even if session variable set.
+    /// If true, it's decided by session variable `streaming_use_shared_source` (default true)
+    pub enable_shared_source: bool,
 }
 
 /// The subsections `[batch.developer]`.
@@ -1355,6 +1426,10 @@ pub mod default {
             3600 * 3
         }
 
+        pub fn gc_history_retention_time_sec() -> u64 {
+            3600 * 6
+        }
+
         pub fn full_gc_interval_sec() -> u64 {
             600
         }
@@ -1423,7 +1498,7 @@ pub mod default {
             1800 // 30mi
         }
 
-        pub fn periodic_scheduling_compaction_group_interval_sec() -> u64 {
+        pub fn periodic_scheduling_compaction_group_split_interval_sec() -> u64 {
             10 // 10s
         }
 
@@ -1431,10 +1506,12 @@ pub mod default {
             600
         }
 
+        // limit the size of state table to trigger split by high throughput
         pub fn move_table_size_limit() -> u64 {
             10 * 1024 * 1024 * 1024 // 10GB
         }
 
+        // limit the size of group to trigger split by group_size and avoid too many small groups
         pub fn split_group_size_limit() -> u64 {
             64 * 1024 * 1024 * 1024 // 64GB
         }
@@ -1443,11 +1520,11 @@ pub mod default {
             16
         }
 
-        pub fn table_write_throughput_threshold() -> u64 {
+        pub fn table_high_write_throughput_threshold() -> u64 {
             16 * 1024 * 1024 // 16MB
         }
 
-        pub fn min_table_split_write_throughput() -> u64 {
+        pub fn table_low_write_throughput_threshold() -> u64 {
             4 * 1024 * 1024 // 4MB
         }
 
@@ -1498,6 +1575,30 @@ pub mod default {
         pub fn enable_dropped_column_reclaim() -> bool {
             false
         }
+
+        pub fn split_group_size_ratio() -> f64 {
+            0.9
+        }
+
+        pub fn table_stat_high_write_throughput_ratio_for_split() -> f64 {
+            0.5
+        }
+
+        pub fn table_stat_low_write_throughput_ratio_for_merge() -> f64 {
+            0.7
+        }
+
+        pub fn table_stat_throuput_window_seconds_for_split() -> usize {
+            60
+        }
+
+        pub fn table_stat_throuput_window_seconds_for_merge() -> usize {
+            240
+        }
+
+        pub fn periodic_scheduling_compaction_group_merge_interval_sec() -> u64 {
+            60 * 10 // 10min
+        }
     }
 
     pub mod server {
@@ -1694,6 +1795,7 @@ pub mod default {
             64
         }
 
+        // deprecated
         pub fn table_info_statistic_history_times() -> usize {
             240
         }
@@ -1956,6 +2058,10 @@ pub mod default {
             true
         }
 
+        pub fn enable_shared_source() -> bool {
+            true
+        }
+
         pub fn stream_high_join_amplification_threshold() -> usize {
             2048
         }
@@ -2253,6 +2359,34 @@ pub mod default {
             }
         }
     }
+
+    pub mod meta_store_config {
+        const DEFAULT_MAX_CONNECTIONS: u32 = 10;
+        const DEFAULT_MIN_CONNECTIONS: u32 = 1;
+        const DEFAULT_CONNECTION_TIMEOUT_SEC: u64 = 10;
+        const DEFAULT_IDLE_TIMEOUT_SEC: u64 = 30;
+        const DEFAULT_ACQUIRE_TIMEOUT_SEC: u64 = 30;
+
+        pub fn max_connections() -> u32 {
+            DEFAULT_MAX_CONNECTIONS
+        }
+
+        pub fn min_connections() -> u32 {
+            DEFAULT_MIN_CONNECTIONS
+        }
+
+        pub fn connection_timeout_sec() -> u64 {
+            DEFAULT_CONNECTION_TIMEOUT_SEC
+        }
+
+        pub fn idle_timeout_sec() -> u64 {
+            DEFAULT_IDLE_TIMEOUT_SEC
+        }
+
+        pub fn acquire_timeout_sec() -> u64 {
+            DEFAULT_ACQUIRE_TIMEOUT_SEC
+        }
+    }
 }
 
 #[derive(Debug, Clone)]
@@ -2467,6 +2601,25 @@ pub struct CompactionConfig {
     pub max_level: u32,
 }
 
+#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)]
+pub struct MetaStoreConfig {
+    /// Maximum number of connections for the meta store connection pool.
+    #[serde(default = "default::meta_store_config::max_connections")]
+    pub max_connections: u32,
+    /// Minimum number of connections for the meta store connection pool.
+    #[serde(default = "default::meta_store_config::min_connections")]
+    pub min_connections: u32,
+    /// Connection timeout in seconds for a meta store connection.
+    #[serde(default = "default::meta_store_config::connection_timeout_sec")]
+    pub connection_timeout_sec: u64,
+    /// Idle timeout in seconds for a meta store connection.
+    #[serde(default = "default::meta_store_config::idle_timeout_sec")]
+    pub idle_timeout_sec: u64,
+    /// Acquire timeout in seconds for a meta store connection.
+    #[serde(default = "default::meta_store_config::acquire_timeout_sec")]
+    pub acquire_timeout_sec: u64,
+}
+
 #[cfg(test)]
 mod tests {
     use risingwave_license::LicenseKey;
@@ -2682,6 +2835,8 @@ mod tests {
                 r#"
             [meta]
             periodic_split_compact_group_interval_sec = 1
+            table_write_throughput_threshold = 10
+            min_table_split_write_throughput = 5
             "#,
             )
             .unwrap();
@@ -2689,9 +2844,11 @@ mod tests {
             assert_eq!(
                 config
                     .meta
-                    .periodic_scheduling_compaction_group_interval_sec,
+                    .periodic_scheduling_compaction_group_split_interval_sec,
                 1
             );
+            assert_eq!(config.meta.table_high_write_throughput_threshold, 10);
+            assert_eq!(config.meta.table_low_write_throughput_threshold, 5);
         }
     }
 }
diff --git a/src/common/src/hash/consistent_hash/bitmap.rs b/src/common/src/hash/consistent_hash/bitmap.rs
index a40946273a0a7..0ba54d8e0637b 100644
--- a/src/common/src/hash/consistent_hash/bitmap.rs
+++ b/src/common/src/hash/consistent_hash/bitmap.rs
@@ -45,16 +45,16 @@ impl Bitmap {
     }
 
     /// Get the reference to a vnode bitmap for singleton actor or table, i.e., with length
-    /// [`VirtualNode::COUNT_FOR_COMPAT`] and only the [`SINGLETON_VNODE`] set to 1.
+    /// 1 and the only [`SINGLETON_VNODE`] set to true.
     pub fn singleton() -> &'static Self {
         Self::singleton_arc()
     }
 
     /// Get the reference to a vnode bitmap for singleton actor or table, i.e., with length
-    /// [`VirtualNode::COUNT_FOR_COMPAT`] and only the [`SINGLETON_VNODE`] set to 1.
+    /// 1 and the only [`SINGLETON_VNODE`] set to true.
     pub fn singleton_arc() -> &'static Arc<Self> {
         static SINGLETON: LazyLock<Arc<Bitmap>> = LazyLock::new(|| {
-            let mut builder = BitmapBuilder::zeroed(VirtualNode::COUNT_FOR_COMPAT);
+            let mut builder = BitmapBuilder::zeroed(1);
             builder.set(SINGLETON_VNODE.to_index(), true);
             builder.finish().into()
         });
diff --git a/src/common/src/hash/consistent_hash/compat.rs b/src/common/src/hash/consistent_hash/compat.rs
deleted file mode 100644
index 0c86fbb12bcd4..0000000000000
--- a/src/common/src/hash/consistent_hash/compat.rs
+++ /dev/null
@@ -1,52 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use super::vnode::VirtualNode;
-
-/// A trait for accessing the vnode count field with backward compatibility.
-pub trait VnodeCountCompat {
-    /// Returns the vnode count, or [`VirtualNode::COUNT_FOR_COMPAT`] if the vnode count is not set,
-    /// typically for backward compatibility.
-    ///
-    /// See the documentation on the field of the implementing type for more details.
-    fn vnode_count(&self) -> usize;
-}
-
-/// Implement the trait for given types by delegating to the `maybe_vnode_count` field.
-///
-/// The reason why there's a `maybe_` prefix is that, a getter method with the same name
-/// as the field will be generated for `prost` structs. Directly naming it `vnode_count`
-/// will lead to the method `vnode_count()` returning `0` when the field is unset, which
-/// can be misleading sometimes.
-///
-/// Instead, we name the field as `maybe_vnode_count` and provide the method `vnode_count`
-/// through this trait, ensuring that backward compatibility is handled properly.
-macro_rules! impl_maybe_vnode_count_compat {
-    ($($ty:ty),* $(,)?) => {
-        $(
-            impl VnodeCountCompat for $ty {
-                fn vnode_count(&self) -> usize {
-                    self.maybe_vnode_count
-                        .map_or(VirtualNode::COUNT_FOR_COMPAT, |v| v as _)
-                }
-            }
-        )*
-    };
-}
-
-impl_maybe_vnode_count_compat!(
-    risingwave_pb::plan_common::StorageTableDesc,
-    risingwave_pb::catalog::Table,
-    risingwave_pb::meta::table_fragments::Fragment,
-);
diff --git a/src/common/src/hash/consistent_hash/mapping.rs b/src/common/src/hash/consistent_hash/mapping.rs
index 1e7bca125fc50..2deed907103f1 100644
--- a/src/common/src/hash/consistent_hash/mapping.rs
+++ b/src/common/src/hash/consistent_hash/mapping.rs
@@ -139,11 +139,12 @@ impl<T: VnodeMappingItem> VnodeMapping<T> {
         }
     }
 
-    /// Create a vnode mapping with the single item. Should only be used for singletons.
+    /// Create a vnode mapping with the single item and length of 1.
     ///
-    /// For backwards compatibility, [`VirtualNode::COUNT_FOR_COMPAT`] is used as the vnode count.
+    /// Should only be used for singletons. If you want a different vnode count, call
+    /// [`VnodeMapping::new_uniform`] with `std::iter::once(item)` and desired length.
     pub fn new_single(item: T::Item) -> Self {
-        Self::new_uniform(std::iter::once(item), VirtualNode::COUNT_FOR_COMPAT)
+        Self::new_uniform(std::iter::once(item), 1)
     }
 
     /// The length (or count) of the vnode in this mapping.
diff --git a/src/common/src/hash/consistent_hash/mod.rs b/src/common/src/hash/consistent_hash/mod.rs
index 98a970af5947d..5f42369248bab 100644
--- a/src/common/src/hash/consistent_hash/mod.rs
+++ b/src/common/src/hash/consistent_hash/mod.rs
@@ -13,6 +13,6 @@
 // limitations under the License.
 
 pub mod bitmap;
-pub mod compat;
 pub mod mapping;
 pub mod vnode;
+pub mod vnode_count;
diff --git a/src/common/src/hash/consistent_hash/vnode_count.rs b/src/common/src/hash/consistent_hash/vnode_count.rs
new file mode 100644
index 0000000000000..0040a79f780fa
--- /dev/null
+++ b/src/common/src/hash/consistent_hash/vnode_count.rs
@@ -0,0 +1,164 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::num::NonZeroUsize;
+
+use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType;
+
+use super::vnode::VirtualNode;
+
+/// The different cases of `maybe_vnode_count` field in the protobuf message.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
+pub enum VnodeCount {
+    /// The field is a placeholder and has to be filled first before using it.
+    #[default]
+    Placeholder,
+    /// The field is set to a specific value.
+    Set(NonZeroUsize),
+    /// The field is unset because the table/fragment is persisted as hash-distributed
+    /// in an older version.
+    CompatHash,
+    /// The field is unset because the table/fragment is persisted as singleton
+    /// in an older version.
+    CompatSingleton,
+}
+
+impl VnodeCount {
+    /// Creates a `VnodeCount` set to the given value.
+    pub fn set(v: impl TryInto<usize> + Copy + std::fmt::Debug) -> Self {
+        let v = (v.try_into().ok())
+            .filter(|v| (1..=VirtualNode::MAX_COUNT).contains(v))
+            .unwrap_or_else(|| panic!("invalid vnode count {v:?}"));
+
+        VnodeCount::Set(NonZeroUsize::new(v).unwrap())
+    }
+
+    /// Creates a `VnodeCount` set to the value for testing.
+    ///
+    /// Equivalent to `VnodeCount::set(VirtualNode::COUNT_FOR_TEST)`.
+    pub fn for_test() -> Self {
+        Self::set(VirtualNode::COUNT_FOR_TEST)
+    }
+
+    /// Converts from protobuf representation of `maybe_vnode_count`. If the value is not set,
+    /// call `compat_is_singleton` to determine whether it should be treated as a singleton
+    /// when it comes to backward compatibility.
+    fn from_protobuf(v: Option<u32>, compat_is_singleton: impl FnOnce() -> bool) -> Self {
+        match v {
+            Some(0) => VnodeCount::Placeholder,
+            Some(v) => VnodeCount::set(v as usize),
+            None => {
+                if compat_is_singleton() {
+                    VnodeCount::CompatSingleton
+                } else {
+                    VnodeCount::CompatHash
+                }
+            }
+        }
+    }
+
+    /// Converts to protobuf representation for `maybe_vnode_count`.
+    pub fn to_protobuf(self) -> Option<u32> {
+        // Effectively fills the compatibility cases with values.
+        self.value_opt()
+            .map_or(Some(0) /* placeholder */, |v| Some(v as _))
+    }
+
+    /// Returns the value of the vnode count, or `None` if it's a placeholder.
+    pub fn value_opt(self) -> Option<usize> {
+        match self {
+            VnodeCount::Placeholder => None,
+            VnodeCount::Set(v) => Some(v.get()),
+            VnodeCount::CompatHash => Some(VirtualNode::COUNT_FOR_COMPAT),
+            VnodeCount::CompatSingleton => Some(1),
+        }
+    }
+
+    /// Returns the value of the vnode count. Panics if it's a placeholder.
+    pub fn value(self) -> usize {
+        self.value_opt()
+            .expect("vnode count is a placeholder that must be filled by the meta service first")
+    }
+}
+
+/// A trait for checking whether a table/fragment is a singleton.
+pub trait IsSingleton {
+    /// Returns `true` if the table/fragment is a singleton.
+    ///
+    /// By singleton, we mean that all data read from or written to the storage belongs to
+    /// the only `SINGLETON_VNODE`. This must be consistent with the behavior of
+    /// [`TableDistribution`](crate::hash::table_distribution::TableDistribution::new).
+    /// As a result, the `vnode_count` of such table/fragment can be `1`.
+    fn is_singleton(&self) -> bool;
+}
+
+/// A trait for accessing the vnode count field with backward compatibility.
+///
+/// # `maybe_`?
+///
+/// The reason why there's a `maybe_` prefix on the protobuf field is that, a getter
+/// method with the same name as the field will be generated for `prost` structs.
+/// Directly naming it `vnode_count` will lead to the method `vnode_count()` returning
+/// `0` when the field is unset, which can be misleading sometimes.
+///
+/// Instead, we name the field as `maybe_vnode_count` and provide the method `vnode_count`
+/// through this trait, ensuring that backward compatibility is handled properly.
+pub trait VnodeCountCompat {
+    /// Get the `maybe_vnode_count` field.
+    fn vnode_count_inner(&self) -> VnodeCount;
+
+    /// Returns the vnode count if it's set. Otherwise, returns [`VirtualNode::COUNT_FOR_COMPAT`]
+    /// for distributed tables/fragments, and `1` for singleton tables/fragments, for backward
+    /// compatibility. Panics if the field is a placeholder.
+    ///
+    /// See the documentation on the field of the implementing type for more details.
+    fn vnode_count(&self) -> usize {
+        self.vnode_count_inner().value()
+    }
+}
+
+impl IsSingleton for risingwave_pb::catalog::Table {
+    fn is_singleton(&self) -> bool {
+        self.distribution_key.is_empty()
+            && self.dist_key_in_pk.is_empty()
+            && self.vnode_col_index.is_none()
+    }
+}
+impl VnodeCountCompat for risingwave_pb::catalog::Table {
+    fn vnode_count_inner(&self) -> VnodeCount {
+        VnodeCount::from_protobuf(self.maybe_vnode_count, || self.is_singleton())
+    }
+}
+
+impl IsSingleton for risingwave_pb::plan_common::StorageTableDesc {
+    fn is_singleton(&self) -> bool {
+        self.dist_key_in_pk_indices.is_empty() && self.vnode_col_idx_in_pk.is_none()
+    }
+}
+impl VnodeCountCompat for risingwave_pb::plan_common::StorageTableDesc {
+    fn vnode_count_inner(&self) -> VnodeCount {
+        VnodeCount::from_protobuf(self.maybe_vnode_count, || self.is_singleton())
+    }
+}
+
+impl IsSingleton for risingwave_pb::meta::table_fragments::Fragment {
+    fn is_singleton(&self) -> bool {
+        matches!(self.distribution_type(), FragmentDistributionType::Single)
+    }
+}
+impl VnodeCountCompat for risingwave_pb::meta::table_fragments::Fragment {
+    fn vnode_count_inner(&self) -> VnodeCount {
+        VnodeCount::from_protobuf(self.maybe_vnode_count, || self.is_singleton())
+    }
+}
diff --git a/src/common/src/hash/key.rs b/src/common/src/hash/key.rs
index 96928e69f4a83..1c6f60f6c621a 100644
--- a/src/common/src/hash/key.rs
+++ b/src/common/src/hash/key.rs
@@ -480,7 +480,7 @@ impl HashKeyDe for Int256 {
     }
 }
 
-impl<'a> HashKeySer<'a> for Serial {
+impl HashKeySer<'_> for Serial {
     fn serialize_into(self, mut buf: impl BufMut) {
         buf.put_i64_ne(self.as_row_id());
     }
diff --git a/src/common/src/hash/mod.rs b/src/common/src/hash/mod.rs
index e54376bed2d50..8ed8120671179 100644
--- a/src/common/src/hash/mod.rs
+++ b/src/common/src/hash/mod.rs
@@ -19,9 +19,9 @@ mod key_v2;
 pub mod table_distribution;
 
 pub use consistent_hash::bitmap::*;
-pub use consistent_hash::compat::*;
 pub use consistent_hash::mapping::*;
 pub use consistent_hash::vnode::*;
+pub use consistent_hash::vnode_count::*;
 pub use dispatcher::{calc_hash_key_kind, HashKeyDispatcher};
 pub use key::{
     Crc32HashCode, HashCode, HashKeyDe, HashKeySer, HeapNullBitmap, NullBitmap,
diff --git a/src/common/src/hash/table_distribution.rs b/src/common/src/hash/table_distribution.rs
index 822db591c1577..a91f48f6589af 100644
--- a/src/common/src/hash/table_distribution.rs
+++ b/src/common/src/hash/table_distribution.rs
@@ -20,7 +20,7 @@ use risingwave_pb::plan_common::StorageTableDesc;
 
 use crate::array::{Array, DataChunk, PrimitiveArray};
 use crate::bitmap::Bitmap;
-use crate::hash::VirtualNode;
+use crate::hash::{IsSingleton, VirtualNode};
 use crate::row::Row;
 use crate::util::iter_util::ZipEqFast;
 
@@ -64,7 +64,10 @@ impl TableDistribution {
             .map(|&k| k as usize)
             .collect_vec();
         let vnode_col_idx_in_pk = table_desc.vnode_col_idx_in_pk.map(|k| k as usize);
-        Self::new(vnodes, dist_key_in_pk_indices, vnode_col_idx_in_pk)
+
+        let this = Self::new(vnodes, dist_key_in_pk_indices, vnode_col_idx_in_pk);
+        assert_eq!(this.is_singleton(), table_desc.is_singleton());
+        this
     }
 
     pub fn new(
diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs
index e3417853b0201..b44aba68b0985 100644
--- a/src/common/src/lib.rs
+++ b/src/common/src/lib.rs
@@ -18,7 +18,6 @@
 )]
 #![feature(extract_if)]
 #![feature(trait_alias)]
-#![feature(is_sorted)]
 #![feature(type_alias_impl_trait)]
 #![feature(test)]
 #![feature(trusted_len)]
@@ -35,7 +34,6 @@
 #![feature(iter_order_by)]
 #![feature(binary_heap_into_iter_sorted)]
 #![feature(impl_trait_in_assoc_type)]
-#![feature(map_entry_replace)]
 #![feature(negative_impls)]
 #![feature(register_tool)]
 #![feature(btree_cursors)]
@@ -93,6 +91,7 @@ pub mod test_utils;
 pub mod transaction;
 pub mod types;
 pub mod vnode_mapping;
+
 pub mod test_prelude {
     pub use super::array::{DataChunkTestExt, StreamChunkTestExt};
     pub use super::catalog::test_utils::ColumnDescTestExt;
diff --git a/src/common/src/row/mod.rs b/src/common/src/row/mod.rs
index 0b2181105352b..2aba5ce074c6d 100644
--- a/src/common/src/row/mod.rs
+++ b/src/common/src/row/mod.rs
@@ -187,7 +187,7 @@ pub trait RowExt: Row {
 
     fn display(&self) -> impl Display + '_ {
         struct D<'a, T: Row>(&'a T);
-        impl<'a, T: Row> Display for D<'a, T> {
+        impl<T: Row> Display for D<'_, T> {
             fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
                 write!(
                     f,
diff --git a/src/common/src/row/project.rs b/src/common/src/row/project.rs
index 542ea6417051b..9c4ee3231b6a4 100644
--- a/src/common/src/row/project.rs
+++ b/src/common/src/row/project.rs
@@ -24,12 +24,12 @@ pub struct Project<'i, R> {
     indices: &'i [usize],
 }
 
-impl<'i, R: Row> PartialEq for Project<'i, R> {
+impl<R: Row> PartialEq for Project<'_, R> {
     fn eq(&self, other: &Self) -> bool {
         self.iter().eq(other.iter())
     }
 }
-impl<'i, R: Row> Eq for Project<'i, R> {}
+impl<R: Row> Eq for Project<'_, R> {}
 
 impl<'i, R: Row> Row for Project<'i, R> {
     #[inline]
diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs
index 43ac67363dd91..4a16fad1bbe96 100644
--- a/src/common/src/session_config/mod.rs
+++ b/src/common/src/session_config/mod.rs
@@ -287,8 +287,8 @@ pub struct SessionConfig {
     ///
     /// When enabled, `CREATE SOURCE` will create a source streaming job, and `CREATE MATERIALIZED VIEWS` from the source
     /// will forward the data from the same source streaming job, and also backfill prior data from the external source.
-    #[parameter(default = false, alias = "rw_enable_shared_source")]
-    enable_shared_source: bool,
+    #[parameter(default = true)]
+    streaming_use_shared_source: bool,
 
     /// Shows the server-side character set encoding. At present, this parameter can be shown but not set, because the encoding is determined at database creation time.
     #[parameter(default = SERVER_ENCODING)]
diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs
index 2343e8e182dd9..4d3a994631534 100644
--- a/src/common/src/system_param/mod.rs
+++ b/src/common/src/system_param/mod.rs
@@ -419,6 +419,17 @@ impl ValidateOnSet for OverrideValidateOnSet {
         }
         Ok(())
     }
+
+    fn time_travel_retention_ms(v: &u64) -> Result<()> {
+        // This is intended to guarantee that non-time-travel batch query can still function even compute node's recent versions doesn't include the desired version.
+        let min_retention_ms = 600_000;
+        if *v < min_retention_ms {
+            return Err(format!(
+                "time_travel_retention_ms cannot be less than {min_retention_ms}"
+            ));
+        }
+        Ok(())
+    }
 }
 
 for_all_params!(impl_default_from_other_params);
diff --git a/src/common/src/types/from_sql.rs b/src/common/src/types/from_sql.rs
index ba1d49892c602..76746bbb871fb 100644
--- a/src/common/src/types/from_sql.rs
+++ b/src/common/src/types/from_sql.rs
@@ -36,10 +36,22 @@ impl<'a> FromSql<'a> for ScalarImpl {
             Type::JSONB => ScalarImpl::from(JsonbVal::from_sql(ty, raw)?),
             Type::INTERVAL => ScalarImpl::from(Interval::from_sql(ty, raw)?),
             Type::BYTEA => ScalarImpl::from(Vec::<u8>::from_sql(ty, raw)?.into_boxed_slice()),
-            Type::VARCHAR | Type::TEXT => ScalarImpl::from(String::from_sql(ty, raw)?),
+            Type::VARCHAR | Type::TEXT | Type::BPCHAR => {
+                ScalarImpl::from(String::from_sql(ty, raw)?)
+            }
+            ref ty
+                if (ty.name() == "citext"
+                    || ty.name() == "ltree"
+                    || ty.name() == "lquery"
+                    || ty.name() == "ltxtquery") =>
+            {
+                ScalarImpl::from(String::from_sql(ty, raw)?)
+            }
             // Serial, Int256, Struct, List and Decimal are not supported here
             // Note: The Decimal type is specially handled in the `ScalarAdapter`.
-            _ => bail_not_implemented!("the postgres decoding for {ty} is unsupported"),
+            _ => {
+                bail_not_implemented!("the postgres decoding for {ty} is unsupported")
+            }
         })
     }
 
@@ -61,6 +73,10 @@ impl<'a> FromSql<'a> for ScalarImpl {
                 | Type::BYTEA
                 | Type::VARCHAR
                 | Type::TEXT
-        )
+                | Type::BPCHAR
+        ) || (ty.name() == "citext"
+            || ty.name() == "ltree"
+            || ty.name() == "lquery"
+            || ty.name() == "ltxtquery")
     }
 }
diff --git a/src/common/src/types/mod.rs b/src/common/src/types/mod.rs
index 54b6c0718d905..44be87116643c 100644
--- a/src/common/src/types/mod.rs
+++ b/src/common/src/types/mod.rs
@@ -951,7 +951,7 @@ impl ScalarImpl {
     }
 }
 
-impl<'a> ScalarRefImpl<'a> {
+impl ScalarRefImpl<'_> {
     /// Converts [`ScalarRefImpl`] to [`ScalarImpl`]
     pub fn into_scalar_impl(self) -> ScalarImpl {
         dispatch_scalar_ref_variants!(self, inner, { inner.to_owned_scalar().into() })
diff --git a/src/common/src/types/num256.rs b/src/common/src/types/num256.rs
index 2acbcf636a037..6c96b3ddbbec8 100644
--- a/src/common/src/types/num256.rs
+++ b/src/common/src/types/num256.rs
@@ -42,7 +42,7 @@ pub struct Int256(pub(crate) Box<i256>);
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd)]
 pub struct Int256Ref<'a>(pub &'a i256);
 
-impl<'a> Display for Int256Ref<'a> {
+impl Display for Int256Ref<'_> {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         self.write(f)
     }
@@ -208,7 +208,7 @@ impl Int256 {
     }
 }
 
-impl<'a> Int256Ref<'a> {
+impl Int256Ref<'_> {
     pub fn memcmp_serialize(
         &self,
         serializer: &mut memcomparable::Serializer<impl bytes::BufMut>,
diff --git a/src/common/src/types/scalar_impl.rs b/src/common/src/types/scalar_impl.rs
index 43742f74c7b51..9384961e4a890 100644
--- a/src/common/src/types/scalar_impl.rs
+++ b/src/common/src/types/scalar_impl.rs
@@ -145,7 +145,7 @@ impl Scalar for bool {
 }
 
 /// Implement `ScalarRef` for `bool`.
-impl<'a> ScalarRef<'a> for bool {
+impl ScalarRef<'_> for bool {
     type ScalarType = bool;
 
     fn to_owned_scalar(&self) -> bool {
@@ -167,7 +167,7 @@ impl Scalar for Decimal {
 }
 
 /// Implement `ScalarRef` for `Decimal`.
-impl<'a> ScalarRef<'a> for Decimal {
+impl ScalarRef<'_> for Decimal {
     type ScalarType = Decimal;
 
     fn to_owned_scalar(&self) -> Decimal {
@@ -189,7 +189,7 @@ impl Scalar for Interval {
 }
 
 /// Implement `ScalarRef` for `Interval`.
-impl<'a> ScalarRef<'a> for Interval {
+impl ScalarRef<'_> for Interval {
     type ScalarType = Interval;
 
     fn to_owned_scalar(&self) -> Interval {
@@ -211,7 +211,7 @@ impl Scalar for Date {
 }
 
 /// Implement `ScalarRef` for `Date`.
-impl<'a> ScalarRef<'a> for Date {
+impl ScalarRef<'_> for Date {
     type ScalarType = Date;
 
     fn to_owned_scalar(&self) -> Date {
@@ -233,7 +233,7 @@ impl Scalar for Timestamp {
 }
 
 /// Implement `ScalarRef` for `Timestamp`.
-impl<'a> ScalarRef<'a> for Timestamp {
+impl ScalarRef<'_> for Timestamp {
     type ScalarType = Timestamp;
 
     fn to_owned_scalar(&self) -> Timestamp {
@@ -255,7 +255,7 @@ impl Scalar for Time {
 }
 
 /// Implement `ScalarRef` for `Time`.
-impl<'a> ScalarRef<'a> for Time {
+impl ScalarRef<'_> for Time {
     type ScalarType = Time;
 
     fn to_owned_scalar(&self) -> Time {
@@ -277,7 +277,7 @@ impl Scalar for Timestamptz {
 }
 
 /// Implement `ScalarRef` for `Timestamptz`.
-impl<'a> ScalarRef<'a> for Timestamptz {
+impl ScalarRef<'_> for Timestamptz {
     type ScalarType = Timestamptz;
 
     fn to_owned_scalar(&self) -> Timestamptz {
@@ -322,7 +322,7 @@ impl ScalarImpl {
     }
 }
 
-impl<'scalar> ScalarRefImpl<'scalar> {
+impl ScalarRefImpl<'_> {
     pub fn get_ident(&self) -> &'static str {
         dispatch_scalar_ref_variants!(self, [I = VARIANT_NAME], { I })
     }
diff --git a/src/common/src/types/with_data_type.rs b/src/common/src/types/with_data_type.rs
index 257d8c7f15dc1..1001a277b316d 100644
--- a/src/common/src/types/with_data_type.rs
+++ b/src/common/src/types/with_data_type.rs
@@ -92,7 +92,7 @@ impl_with_data_type!(rust_decimal::Decimal, DataType::Decimal);
 impl_with_data_type!(Decimal, DataType::Decimal);
 impl_with_data_type!(Serial, DataType::Serial);
 
-impl<'a> WithDataType for &'a str {
+impl WithDataType for &str {
     fn default_data_type() -> DataType {
         DataType::Varchar
     }
@@ -109,7 +109,7 @@ impl_with_data_type!(Vec<u8>, DataType::Bytea);
 impl_with_data_type!(Bytes, DataType::Bytea);
 impl_with_data_type!(JsonbVal, DataType::Jsonb);
 
-impl<'a> WithDataType for JsonbRef<'a> {
+impl WithDataType for JsonbRef<'_> {
     fn default_data_type() -> DataType {
         DataType::Jsonb
     }
diff --git a/src/common/src/util/hash_util.rs b/src/common/src/util/hash_util.rs
index 264e4783786bb..85654580827e0 100644
--- a/src/common/src/util/hash_util.rs
+++ b/src/common/src/util/hash_util.rs
@@ -15,10 +15,10 @@
 use std::hash::{BuildHasher, Hasher};
 
 pub fn finalize_hashers<H: Hasher>(hashers: &[H]) -> Vec<u64> {
-    return hashers
+    hashers
         .iter()
         .map(|hasher| hasher.finish())
-        .collect::<Vec<u64>>();
+        .collect::<Vec<u64>>()
 }
 
 #[derive(Clone, Copy)]
diff --git a/src/common/src/util/prost.rs b/src/common/src/util/prost.rs
index 8145a37a8a202..4ea799f1befeb 100644
--- a/src/common/src/util/prost.rs
+++ b/src/common/src/util/prost.rs
@@ -32,7 +32,7 @@ impl TypeUrl for batch_plan::ExchangeNode {
 
 pub struct StackTraceResponseOutput<'a>(&'a StackTraceResponse);
 
-impl<'a> Deref for StackTraceResponseOutput<'a> {
+impl Deref for StackTraceResponseOutput<'_> {
     type Target = StackTraceResponse;
 
     fn deref(&self) -> &Self::Target {
@@ -40,7 +40,7 @@ impl<'a> Deref for StackTraceResponseOutput<'a> {
     }
 }
 
-impl<'a> Display for StackTraceResponseOutput<'a> {
+impl Display for StackTraceResponseOutput<'_> {
     fn fmt(&self, s: &mut Formatter<'_>) -> std::fmt::Result {
         if !self.actor_traces.is_empty() {
             writeln!(s, "--- Actor Traces ---")?;
diff --git a/src/common/src/util/recursive.rs b/src/common/src/util/recursive.rs
index 2869b3c496335..839bcb72e5f3b 100644
--- a/src/common/src/util/recursive.rs
+++ b/src/common/src/util/recursive.rs
@@ -82,7 +82,7 @@ impl Tracker {
             }
         }
 
-        impl<'a> Drop for DepthGuard<'a> {
+        impl Drop for DepthGuard<'_> {
             fn drop(&mut self) {
                 let mut d = self.depth.borrow_mut();
                 d.last_max = d.last_max.max(d.current); // update the last max depth
diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml
index 6e6174616e530..65848b5619ba2 100644
--- a/src/compute/Cargo.toml
+++ b/src/compute/Cargo.toml
@@ -57,7 +57,7 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
 ] }
 tokio-stream = { workspace = true }
 tonic = { workspace = true }
-tower = { version = "0.4", features = ["util", "load-shed"] }
+tower = { version = "0.5", features = ["util", "load-shed"] }
 tracing = "0.1"
 uuid = { version = "1.11.0", features = ["v4"] }
 
diff --git a/src/compute/src/memory/config.rs b/src/compute/src/memory/config.rs
index fdba52884f47e..b74da7d9c481f 100644
--- a/src/compute/src/memory/config.rs
+++ b/src/compute/src/memory/config.rs
@@ -37,7 +37,6 @@ const COMPACTOR_MEMORY_PROPORTION: f64 = 0.1;
 
 const STORAGE_BLOCK_CACHE_MEMORY_PROPORTION: f64 = 0.3;
 
-const STORAGE_META_CACHE_MAX_MEMORY_MB: usize = 4096;
 const STORAGE_SHARED_BUFFER_MAX_MEMORY_MB: usize = 4096;
 const STORAGE_META_CACHE_MEMORY_PROPORTION: f64 = 0.35;
 const STORAGE_SHARED_BUFFER_MEMORY_PROPORTION: f64 = 0.3;
@@ -66,6 +65,14 @@ pub fn reserve_memory_bytes(opts: &ComputeNodeOpts) -> (usize, usize) {
     // Should have at least `MIN_SYSTEM_RESERVED_MEMORY_MB` for reserved memory.
     let reserved = std::cmp::max(reserved, MIN_SYSTEM_RESERVED_MEMORY_MB << 20);
 
+    if reserved >= opts.total_memory_bytes {
+        panic!(
+            "reserved memory ({}) >= total memory ({}).",
+            convert(reserved as _),
+            convert(opts.total_memory_bytes as _)
+        );
+    }
+
     (reserved, opts.total_memory_bytes - reserved)
 }
 
@@ -107,24 +114,60 @@ pub fn storage_memory_config(
     } else {
         (STORAGE_MEMORY_PROPORTION + COMPACTOR_MEMORY_PROPORTION, 0.0)
     };
-    let mut default_block_cache_capacity_mb = ((non_reserved_memory_bytes as f64
-        * storage_memory_proportion
-        * STORAGE_BLOCK_CACHE_MEMORY_PROPORTION)
-        .ceil() as usize)
-        >> 20;
-    let default_meta_cache_capacity_mb = ((non_reserved_memory_bytes as f64
-        * storage_memory_proportion
-        * STORAGE_META_CACHE_MEMORY_PROPORTION)
-        .ceil() as usize)
-        >> 20;
-    let meta_cache_capacity_mb = storage_config.cache.meta_cache_capacity_mb.unwrap_or(
+
+    let storage_memory_bytes = non_reserved_memory_bytes as f64 * storage_memory_proportion;
+
+    // Only if the all cache capacities are specified and their sum doesn't exceed the max allowed storage_memory_bytes, will we use them.
+    // Other invalid combination of the cache capacities config will be ignored.
+    let (
+        config_block_cache_capacity_mb,
+        config_meta_cache_capacity_mb,
+        config_shared_buffer_capacity_mb,
+    ) = match (
+        storage_config.cache.block_cache_capacity_mb,
+        storage_config.cache.meta_cache_capacity_mb,
+        storage_config.shared_buffer_capacity_mb,
+    ) {
+        (
+            Some(block_cache_capacity_mb),
+            Some(meta_cache_capacity_mb),
+            Some(shared_buffer_capacity_mb),
+        ) => {
+            let config_storage_memory_bytes =
+                (block_cache_capacity_mb + meta_cache_capacity_mb + shared_buffer_capacity_mb)
+                    << 20;
+            if config_storage_memory_bytes as f64 > storage_memory_bytes {
+                tracing::warn!(
+                    "config block_cache_capacity_mb {} + meta_cache_capacity_mb {} + shared_buffer_capacity_mb {} = {} exceeds allowed storage_memory_bytes {}. These configs will be ignored.",
+                    block_cache_capacity_mb, meta_cache_capacity_mb, shared_buffer_capacity_mb, convert(config_storage_memory_bytes as _), convert(storage_memory_bytes as _)
+                );
+                (None, None, None)
+            } else {
+                (
+                    Some(block_cache_capacity_mb),
+                    Some(meta_cache_capacity_mb),
+                    Some(shared_buffer_capacity_mb),
+                )
+            }
+        }
+        c => {
+            tracing::warn!(
+                "config (block_cache_capacity_mb, meta_cache_capacity_mb, shared_buffer_capacity_mb): {:?} should be set altogether. These configs will be ignored.",
+                c
+            );
+            (None, None, None)
+        }
+    };
+
+    let mut default_block_cache_capacity_mb =
+        ((storage_memory_bytes * STORAGE_BLOCK_CACHE_MEMORY_PROPORTION).ceil() as usize) >> 20;
+    let default_meta_cache_capacity_mb =
+        ((storage_memory_bytes * STORAGE_META_CACHE_MEMORY_PROPORTION).ceil() as usize) >> 20;
+    let meta_cache_capacity_mb = config_meta_cache_capacity_mb.unwrap_or(
         // adapt to old version
         storage_config
             .meta_cache_capacity_mb
-            .unwrap_or(std::cmp::min(
-                default_meta_cache_capacity_mb,
-                STORAGE_META_CACHE_MAX_MEMORY_MB,
-            )),
+            .unwrap_or(default_meta_cache_capacity_mb),
     );
 
     let prefetch_buffer_capacity_mb = storage_config
@@ -137,18 +180,12 @@ pub fn storage_memory_config(
             default_block_cache_capacity_mb.saturating_sub(meta_cache_capacity_mb);
     }
 
-    let default_shared_buffer_capacity_mb = ((non_reserved_memory_bytes as f64
-        * storage_memory_proportion
-        * STORAGE_SHARED_BUFFER_MEMORY_PROPORTION)
-        .ceil() as usize)
-        >> 20;
-    let mut shared_buffer_capacity_mb =
-        storage_config
-            .shared_buffer_capacity_mb
-            .unwrap_or(std::cmp::min(
-                default_shared_buffer_capacity_mb,
-                STORAGE_SHARED_BUFFER_MAX_MEMORY_MB,
-            ));
+    let default_shared_buffer_capacity_mb =
+        ((storage_memory_bytes * STORAGE_SHARED_BUFFER_MEMORY_PROPORTION).ceil() as usize) >> 20;
+    let mut shared_buffer_capacity_mb = config_shared_buffer_capacity_mb.unwrap_or(std::cmp::min(
+        default_shared_buffer_capacity_mb,
+        STORAGE_SHARED_BUFFER_MAX_MEMORY_MB,
+    ));
     if is_serving {
         default_block_cache_capacity_mb += default_shared_buffer_capacity_mb;
         // set 1 to pass internal check
@@ -158,7 +195,7 @@ pub fn storage_memory_config(
         default_block_cache_capacity_mb =
             default_block_cache_capacity_mb.saturating_sub(shared_buffer_capacity_mb);
     }
-    let block_cache_capacity_mb = storage_config.cache.block_cache_capacity_mb.unwrap_or(
+    let block_cache_capacity_mb = config_block_cache_capacity_mb.unwrap_or(
         // adapt to old version
         storage_config
             .block_cache_capacity_mb
@@ -353,6 +390,7 @@ mod tests {
 
         let total_non_reserved_memory_bytes = 8 << 30;
 
+        // Embedded compactor enabled, streaming node
         let memory_config = storage_memory_config(
             total_non_reserved_memory_bytes,
             true,
@@ -364,6 +402,7 @@ mod tests {
         assert_eq!(memory_config.shared_buffer_capacity_mb, 737);
         assert_eq!(memory_config.compactor_memory_limit_mb, 819);
 
+        // Embedded compactor disabled, serving node
         let memory_config = storage_memory_config(
             total_non_reserved_memory_bytes,
             false,
@@ -375,6 +414,7 @@ mod tests {
         assert_eq!(memory_config.shared_buffer_capacity_mb, 1);
         assert_eq!(memory_config.compactor_memory_limit_mb, 0);
 
+        // Embedded compactor enabled, streaming node, file cache
         storage_config.data_file_cache.dir = "data".to_string();
         storage_config.meta_file_cache.dir = "meta".to_string();
         let memory_config = storage_memory_config(
@@ -388,11 +428,17 @@ mod tests {
         assert_eq!(memory_config.shared_buffer_capacity_mb, 737);
         assert_eq!(memory_config.compactor_memory_limit_mb, 819);
 
+        // Embedded compactor enabled, streaming node, file cache, cache capacities specified
         storage_config.cache.block_cache_capacity_mb = Some(512);
         storage_config.cache.meta_cache_capacity_mb = Some(128);
         storage_config.shared_buffer_capacity_mb = Some(1024);
         storage_config.compactor_memory_limit_mb = Some(512);
-        let memory_config = storage_memory_config(0, true, &storage_config, false);
+        let memory_config = storage_memory_config(
+            total_non_reserved_memory_bytes,
+            true,
+            &storage_config,
+            false,
+        );
         assert_eq!(memory_config.block_cache_capacity_mb, 512);
         assert_eq!(memory_config.meta_cache_capacity_mb, 128);
         assert_eq!(memory_config.shared_buffer_capacity_mb, 1024);
diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs
index 909b4f96b7a14..da3328b0b2ced 100644
--- a/src/compute/src/server.rs
+++ b/src/compute/src/server.rs
@@ -256,7 +256,7 @@ pub async fn compute_node_serve(
                 compactor_context,
                 hummock_meta_client.clone(),
                 storage.sstable_object_id_manager().clone(),
-                storage.filter_key_extractor_manager().clone(),
+                storage.compaction_catalog_manager_ref().clone(),
             );
             sub_tasks.push((handle, shutdown_sender));
         }
diff --git a/src/config/ci-backfill.toml b/src/config/ci-backfill.toml
index 7d0660e8377fe..55b6149df3ebb 100644
--- a/src/config/ci-backfill.toml
+++ b/src/config/ci-backfill.toml
@@ -5,7 +5,7 @@ telemetry_enabled = false
 stream_chunk_size = 10
 
 [storage.object_store.retry]
-streaming_upload_attempt_timeout_ms = 30000
+streaming_upload_attempt_timeout_ms = 20000
 
 [system]
 # NOTE(kwannoel): If can't reproduce it, set to a lower number.
diff --git a/src/config/ci-longer-streaming-upload-timeout.toml b/src/config/ci-longer-streaming-upload-timeout.toml
index 5510ffc51c17e..90bd3790527d2 100644
--- a/src/config/ci-longer-streaming-upload-timeout.toml
+++ b/src/config/ci-longer-streaming-upload-timeout.toml
@@ -17,7 +17,7 @@ stream_exchange_concurrent_barriers = 10
 imm_merge_threshold = 2
 
 [storage.object_store.retry]
-streaming_upload_attempt_timeout_ms = 30000
+streaming_upload_attempt_timeout_ms = 20000
 upload_retry_attempts = 5
 read_attempt_timeout_ms = 16000
 read_retry_attempts = 6
diff --git a/src/config/ci-time-travel.toml b/src/config/ci-time-travel.toml
index d160a8c94c69a..491d2bdecac0a 100644
--- a/src/config/ci-time-travel.toml
+++ b/src/config/ci-time-travel.toml
@@ -2,4 +2,4 @@
 hummock_time_travel_snapshot_interval = 30
 
 [system]
-time_travel_retention_ms = 300000
+time_travel_retention_ms = 600000
diff --git a/src/config/docs.md b/src/config/docs.md
index 6e5a1359ca671..4a25867ed63c1 100644
--- a/src/config/docs.md
+++ b/src/config/docs.md
@@ -39,6 +39,7 @@ This page is automatically generated by `./risedev generate-example-config`
 | event_log_enabled |  | true |
 | full_gc_interval_sec | Interval of automatic hummock full GC. | 600 |
 | full_gc_object_limit | Max number of object per full GC job can fetch. | 100000 |
+| gc_history_retention_time_sec | Duration in seconds to retain garbage collection history data. | 21600 |
 | hummock_time_travel_snapshot_interval | The interval at which a Hummock version snapshot is taken for time travel. Larger value indicates less storage overhead but worse query performance. | 100 |
 | hummock_version_checkpoint_interval_sec | Interval of hummock version checkpoint. | 30 |
 | hybrid_partition_vnode_count | Count of partitions of tables in default group and materialized view group. The meta node will decide according to some strategy whether to cut the boundaries of the file according to the vnode alignment. Each partition contains aligned data of `vnode_count / hybrid_partition_vnode_count` consecutive virtual-nodes of one state table. Set it zero to disable this feature. | 4 |
@@ -47,7 +48,6 @@ This page is automatically generated by `./risedev generate-example-config`
 | meta_leader_lease_secs |  | 30 |
 | min_delta_log_num_for_hummock_version_checkpoint | The minimum delta log number a new checkpoint should compact, otherwise the checkpoint attempt is rejected. | 10 |
 | min_sst_retention_time_sec | Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they are dangling. | 10800 |
-| min_table_split_write_throughput | If the size of one table is smaller than `min_table_split_write_throughput`, we would not split it to an single group. | 4194304 |
 | move_table_size_limit |  | 10737418240 |
 | node_num_monitor_interval_sec |  | 10 |
 | parallelism_control_batch_size | The number of streaming jobs per scaling operation. | 10 |
@@ -55,13 +55,19 @@ This page is automatically generated by `./risedev generate-example-config`
 | parallelism_control_trigger_period_sec | The period of parallelism control trigger. | 10 |
 | partition_vnode_count | Count of partition in split group. Meta will assign this value to every new group when it splits from default-group by automatically. Each partition contains aligned data of `vnode_count / partition_vnode_count` consecutive virtual-nodes of one state table. | 16 |
 | periodic_compaction_interval_sec | Schedule compaction for all compaction groups with this interval. | 60 |
-| periodic_scheduling_compaction_group_interval_sec |  | 10 |
+| periodic_scheduling_compaction_group_merge_interval_sec |  | 600 |
+| periodic_scheduling_compaction_group_split_interval_sec |  | 10 |
 | periodic_space_reclaim_compaction_interval_sec | Schedule `space_reclaim` compaction for all compaction groups with this interval. | 3600 |
 | periodic_tombstone_reclaim_compaction_interval_sec |  | 600 |
 | periodic_ttl_reclaim_compaction_interval_sec | Schedule `ttl_reclaim` compaction for all compaction groups with this interval. | 1800 |
 | split_group_size_limit |  | 68719476736 |
-| table_write_throughput_threshold | The threshold of write throughput to trigger a group split. Increase this configuration value to avoid split too many groups with few data write. | 16777216 |
-| unrecognized |  |  |
+| split_group_size_ratio | Whether to split the compaction group when the size of the group exceeds the `compaction_group_config.max_estimated_group_size() * split_group_size_ratio`. | 0.9 |
+| table_high_write_throughput_threshold | The threshold of write throughput to trigger a group split. | 16777216 |
+| table_low_write_throughput_threshold | The threshold of write throughput to trigger a group merge. | 4194304 |
+| table_stat_high_write_throughput_ratio_for_split | To split the compaction group when the high throughput statistics of the group exceeds the threshold. | 0.5 |
+| table_stat_low_write_throughput_ratio_for_merge | To merge the compaction group when the low throughput statistics of the group exceeds the threshold. | 0.7 |
+| table_stat_throuput_window_seconds_for_merge | The window seconds of table throughput statistic history for merge compaction group. | 240 |
+| table_stat_throuput_window_seconds_for_split | The window seconds of table throughput statistic history for split compaction group. | 60 |
 | vacuum_interval_sec | Interval of invoking a vacuum job, to remove stale metadata from meta store and objects from object store. | 30 |
 | vacuum_spin_interval_ms | The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of meta node. | 200 |
 
@@ -86,6 +92,16 @@ This page is automatically generated by `./risedev generate-example-config`
 | target_file_size_base |  | 33554432 |
 | tombstone_reclaim_ratio |  | 40 |
 
+## meta.meta_store_config
+
+| Config | Description | Default |
+|--------|-------------|---------|
+| acquire_timeout_sec | Acquire timeout in seconds for a meta store connection. | 30 |
+| connection_timeout_sec | Connection timeout in seconds for a meta store connection. | 10 |
+| idle_timeout_sec | Idle timeout in seconds for a meta store connection. | 30 |
+| max_connections | Maximum number of connections for the meta store connection pool. | 10 |
+| min_connections | Minimum number of connections for the meta store connection pool. | 1 |
+
 ## server
 
 | Config | Description | Default |
@@ -102,8 +118,6 @@ This page is automatically generated by `./risedev generate-example-config`
 | Config | Description | Default |
 |--------|-------------|---------|
 | block_cache_capacity_mb | DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_capacity_mb` instead. |  |
-| cache |  |  |
-| cache_refill |  |  |
 | check_compaction_result |  | false |
 | compact_iter_recreate_timeout_ms |  | 600000 |
 | compactor_concurrent_uploading_sst_count | The concurrent uploading number of `SSTables` of buidler |  |
@@ -116,7 +130,6 @@ This page is automatically generated by `./risedev generate-example-config`
 | compactor_max_task_multiplier | Compactor calculates the maximum number of tasks that can be executed on the node based on `worker_num` and `compactor_max_task_multiplier`. `max_pull_task_count` = `worker_num` * `compactor_max_task_multiplier` | 3.0 |
 | compactor_memory_available_proportion | The percentage of memory available when compactor is deployed separately. `non_reserved_memory_bytes` = `system_memory_available_bytes` * `compactor_memory_available_proportion` | 0.8 |
 | compactor_memory_limit_mb |  |  |
-| data_file_cache |  |  |
 | disable_remote_compactor |  | false |
 | enable_fast_compaction |  | true |
 | high_priority_ratio_in_percent | DEPRECATED: This config will be deprecated in the future version, use `storage.cache.block_cache_eviction.high_priority_ratio_in_percent` with `storage.cache.block_cache_eviction.algorithm = "Lru"` instead. |  |
@@ -129,7 +142,6 @@ This page is automatically generated by `./risedev generate-example-config`
 | max_version_pinning_duration_sec |  | 10800 |
 | mem_table_spill_threshold | The spill threshold for mem table. | 4194304 |
 | meta_cache_capacity_mb | DEPRECATED: This config will be deprecated in the future version, use `storage.cache.meta_cache_capacity_mb` instead. |  |
-| meta_file_cache |  |  |
 | min_sst_size_for_streaming_upload | Whether to enable streaming upload for sstable. | 33554432 |
 | min_sstable_size_mb |  | 32 |
 | object_store | Object storage configuration 1. General configuration 2. Some special configuration of Backend 3. Retry and timeout configuration |  |
@@ -137,14 +149,69 @@ This page is automatically generated by `./risedev generate-example-config`
 | share_buffer_compaction_worker_threads_number | Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use tokio's default value (number of CPU core). | 4 |
 | share_buffer_upload_concurrency | Number of tasks shared buffer can upload in parallel. | 8 |
 | share_buffers_sync_parallelism | parallelism while syncing share buffers into L0 SST. Should NOT be 0. | 1 |
-| shared_buffer_capacity_mb | Maximum shared buffer size, writes attempting to exceed the capacity will stall until there is enough space. |  |
+| shared_buffer_capacity_mb | Configure the maximum shared buffer size in MB explicitly. Writes attempting to exceed the capacity will stall until there is enough space. The overridden value will only be effective if: 1. `block_cache_capacity_mb` and `meta_cache_capacity_mb` are also configured explicitly. 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory. |  |
 | shared_buffer_flush_ratio | The shared buffer will start flushing data to object when the ratio of memory usage to the shared buffer capacity exceed such ratio. | 0.800000011920929 |
 | shared_buffer_min_batch_flush_size_mb | The minimum total flush size of shared buffer spill. When a shared buffer spilled is trigger, the total flush size across multiple epochs should be at least higher than this size. | 800 |
 | sstable_id_remote_fetch_number | Number of SST ids fetched from meta per RPC | 10 |
-| table_info_statistic_history_times | The window size of table info statistic history. | 240 |
+| table_info_statistic_history_times | Deprecated: The window size of table info statistic history. | 240 |
 | time_travel_version_cache_capacity |  | 32 |
 | write_conflict_detection_enabled | Whether to enable write conflict detection | true |
 
+## storage.cache
+
+| Config | Description | Default |
+|--------|-------------|---------|
+| block_cache_capacity_mb | Configure the capacity of the block cache in MB explicitly. The overridden value will only be effective if: 1. `meta_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly. 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory. |  |
+| block_cache_shard_num | Configure the number of shards in the block cache explicitly. If not set, the shard number will be determined automatically based on cache capacity. |  |
+| meta_cache_capacity_mb | Configure the capacity of the block cache in MB explicitly. The overridden value will only be effective if: 1. `block_cache_capacity_mb` and `shared_buffer_capacity_mb` are also configured explicitly. 2. `block_cache_capacity_mb` + `meta_cache_capacity_mb` + `meta_cache_capacity_mb` doesn't exceed 0.3 * non-reserved memory. |  |
+| meta_cache_shard_num | Configure the number of shards in the meta cache explicitly. If not set, the shard number will be determined automatically based on cache capacity. |  |
+
+## storage.cache_refill
+
+| Config | Description | Default |
+|--------|-------------|---------|
+| concurrency | Inflight data cache refill tasks. | 10 |
+| data_refill_levels | `SSTable` levels to refill. | [] |
+| recent_filter_layers | Recent filter layer count. | 6 |
+| recent_filter_rotate_interval_ms | Recent filter layer rotate interval. | 10000 |
+| threshold | Data cache refill unit admission ratio. Only unit whose blocks are admitted above the ratio will be refilled. | 0.5 |
+| timeout_ms | Cache refill maximum timeout to apply version delta. | 6000 |
+| unit | Block count that a data cache refill request fetches. | 64 |
+
+## storage.data_file_cache
+
+| Config | Description | Default |
+|--------|-------------|---------|
+| capacity_mb |  | 1024 |
+| compression |  | "None" |
+| dir |  | "" |
+| file_capacity_mb |  | 64 |
+| flush_buffer_threshold_mb |  |  |
+| flushers |  | 4 |
+| indexer_shards |  | 64 |
+| insert_rate_limit_mb |  | 0 |
+| reclaimers |  | 4 |
+| recover_concurrency |  | 8 |
+| recover_mode | Recover mode. Options: - "None": Do not recover disk cache. - "Quiet": Recover disk cache and skip errors. - "Strict": Recover disk cache and panic on errors. More details, see [`RecoverMode::None`], [`RecoverMode::Quiet`] and [`RecoverMode::Strict`], | "None" |
+| runtime_config |  |  |
+
+## storage.meta_file_cache
+
+| Config | Description | Default |
+|--------|-------------|---------|
+| capacity_mb |  | 1024 |
+| compression |  | "None" |
+| dir |  | "" |
+| file_capacity_mb |  | 64 |
+| flush_buffer_threshold_mb |  |  |
+| flushers |  | 4 |
+| indexer_shards |  | 64 |
+| insert_rate_limit_mb |  | 0 |
+| reclaimers |  | 4 |
+| recover_concurrency |  | 8 |
+| recover_mode | Recover mode. Options: - "None": Do not recover disk cache. - "Quiet": Recover disk cache and skip errors. - "Strict": Recover disk cache and panic on errors. More details, see [`RecoverMode::None`], [`RecoverMode::Quiet`] and [`RecoverMode::Strict`], | "None" |
+| runtime_config |  |  |
+
 ## streaming
 
 | Config | Description | Default |
diff --git a/src/config/example.toml b/src/config/example.toml
index 0d01a09d12875..1d540b429a876 100644
--- a/src/config/example.toml
+++ b/src/config/example.toml
@@ -17,6 +17,7 @@ dir = "./"
 min_sst_retention_time_sec = 10800
 full_gc_interval_sec = 600
 full_gc_object_limit = 100000
+gc_history_retention_time_sec = 21600
 max_inflight_time_travel_query = 1000
 periodic_compaction_interval_sec = 60
 vacuum_interval_sec = 30
@@ -40,22 +41,28 @@ backend = "Mem"
 periodic_space_reclaim_compaction_interval_sec = 3600
 periodic_ttl_reclaim_compaction_interval_sec = 1800
 periodic_tombstone_reclaim_compaction_interval_sec = 600
-periodic_scheduling_compaction_group_interval_sec = 10
 move_table_size_limit = 10737418240
 split_group_size_limit = 68719476736
 cut_table_size_limit = 1073741824
 do_not_config_object_storage_lifecycle = false
 partition_vnode_count = 16
-table_write_throughput_threshold = 16777216
-min_table_split_write_throughput = 4194304
+table_high_write_throughput_threshold = 16777216
+table_low_write_throughput_threshold = 4194304
 compaction_task_max_heartbeat_interval_secs = 30
 compaction_task_max_progress_interval_secs = 600
 hybrid_partition_vnode_count = 4
-compact_task_table_size_partition_threshold_low = 134217728
-compact_task_table_size_partition_threshold_high = 536870912
 event_log_enabled = true
 event_log_channel_max_size = 10
 enable_dropped_column_reclaim = false
+split_group_size_ratio = 0.9
+table_stat_high_write_throughput_ratio_for_split = 0.5
+table_stat_low_write_throughput_ratio_for_merge = 0.7
+table_stat_throuput_window_seconds_for_split = 60
+table_stat_throuput_window_seconds_for_merge = 240
+compact_task_table_size_partition_threshold_low = 134217728
+compact_task_table_size_partition_threshold_high = 536870912
+periodic_scheduling_compaction_group_split_interval_sec = 10
+periodic_scheduling_compaction_group_merge_interval_sec = 600
 
 [meta.compaction_config]
 max_bytes_for_level_base = 536870912
@@ -86,6 +93,13 @@ meta_actor_cnt_per_worker_parallelism_soft_limit = 100
 meta_actor_cnt_per_worker_parallelism_hard_limit = 400
 meta_hummock_time_travel_sst_info_fetch_batch_size = 10000
 
+[meta.meta_store_config]
+max_connections = 10
+min_connections = 1
+connection_timeout_sec = 10
+idle_timeout_sec = 30
+acquire_timeout_sec = 30
+
 [batch]
 enable_barrier_read = false
 statement_timeout_in_sec = 3600
@@ -129,6 +143,7 @@ stream_high_join_amplification_threshold = 2048
 stream_enable_actor_tokio_metrics = false
 stream_exchange_connection_pool_size = 1
 stream_enable_auto_schema_change = true
+stream_enable_shared_source = true
 
 [storage]
 share_buffers_sync_parallelism = 1
diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml
index 79764508e22c4..3fe5499e41d08 100644
--- a/src/connector/Cargo.toml
+++ b/src/connector/Cargo.toml
@@ -50,7 +50,7 @@ csv = "1.3"
 deltalake = { workspace = true }
 duration-str = "0.11.2"
 easy-ext = "1"
-elasticsearch = { version = "8.5.0-alpha.1", features = ["rustls-tls"] }
+elasticsearch = { version = "8.15.0-alpha.1", features = ["rustls-tls"] }
 enum-as-inner = "0.6"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
@@ -71,11 +71,9 @@ jni = { version = "0.21.1", features = ["invocation"] }
 jsonbb = { workspace = true }
 jsonwebtoken = "9.2.0"
 maplit = "1.0.2"
-moka = { version = "0.12.0", features = ["future"] }
+moka = { version = "0.12.8", features = ["future"] }
 mongodb = { version = "2.8.2", features = ["tokio-runtime"] }
-mysql_async = { version = "0.34", default-features = false, features = [
-    "default",
-] }
+mysql_async = { workspace = true }
 mysql_common = { version = "0.32", default-features = false, features = [
     "chrono",
 ] }
@@ -130,7 +128,7 @@ rustls-native-certs = "0.7"
 rustls-pemfile = "2"
 rustls-pki-types = "1"
 rw_futures_util = { workspace = true }
-sea-schema = { version = "0.15", default-features = false, features = [
+sea-schema = { version = "0.16", default-features = false, features = [
     "discovery",
     "sqlx-postgres",
     "sqlx-mysql",
@@ -139,7 +137,7 @@ serde = { version = "1", features = ["derive", "rc"] }
 serde_derive = "1"
 serde_json = "1"
 serde_with = { version = "3", features = ["json"] }
-simd-json = { version = "0.13.3", features = ["hints"] }
+simd-json = { version = "0.14.2", features = ["hints"] }
 sqlx = { workspace = true }
 strum = "0.26"
 strum_macros = "0.26"
diff --git a/src/connector/codec/tests/integration_tests/utils.rs b/src/connector/codec/tests/integration_tests/utils.rs
index 889dbeffc306f..021913edf033d 100644
--- a/src/connector/codec/tests/integration_tests/utils.rs
+++ b/src/connector/codec/tests/integration_tests/utils.rs
@@ -23,7 +23,7 @@ use risingwave_pb::plan_common::AdditionalColumn;
 /// More concise display for `DataType`, to use in tests.
 pub struct DataTypeTestDisplay<'a>(pub &'a DataType);
 
-impl<'a> std::fmt::Debug for DataTypeTestDisplay<'a> {
+impl std::fmt::Debug for DataTypeTestDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self.0 {
             DataType::Struct(s) => {
@@ -68,7 +68,7 @@ impl<'a> std::fmt::Debug for DataTypeTestDisplay<'a> {
 /// More concise display for `ScalarRefImpl`, to use in tests.
 pub struct ScalarRefImplTestDisplay<'a>(pub ScalarRefImpl<'a>);
 
-impl<'a> std::fmt::Debug for ScalarRefImplTestDisplay<'a> {
+impl std::fmt::Debug for ScalarRefImplTestDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self.0 {
             ScalarRefImpl::Struct(s) => {
@@ -112,7 +112,7 @@ impl<'a> std::fmt::Debug for ScalarRefImplTestDisplay<'a> {
 /// More concise display for `ScalarImpl`, to use in tests.
 pub struct ScalarImplTestDisplay<'a>(pub &'a ScalarImpl);
 
-impl<'a> std::fmt::Debug for ScalarImplTestDisplay<'a> {
+impl std::fmt::Debug for ScalarImplTestDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         ScalarRefImplTestDisplay(self.0.as_scalar_ref_impl()).fmt(f)
     }
@@ -121,7 +121,7 @@ impl<'a> std::fmt::Debug for ScalarImplTestDisplay<'a> {
 /// More concise display for `DatumRef`, to use in tests.
 pub struct DatumRefTestDisplay<'a>(pub DatumRef<'a>);
 
-impl<'a> std::fmt::Debug for DatumRefTestDisplay<'a> {
+impl std::fmt::Debug for DatumRefTestDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self.0 {
             Some(scalar) => ScalarRefImplTestDisplay(scalar).fmt(f),
@@ -133,7 +133,7 @@ impl<'a> std::fmt::Debug for DatumRefTestDisplay<'a> {
 /// More concise display for `Datum`, to use in tests.
 pub struct DatumTestDisplay<'a>(pub &'a Datum);
 
-impl<'a> std::fmt::Debug for DatumTestDisplay<'a> {
+impl std::fmt::Debug for DatumTestDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         DatumRefTestDisplay(self.0.to_datum_ref()).fmt(f)
     }
@@ -142,7 +142,7 @@ impl<'a> std::fmt::Debug for DatumTestDisplay<'a> {
 /// More concise display for `DatumCow`, to use in tests.
 pub struct DatumCowTestDisplay<'a>(pub &'a DatumCow<'a>);
 
-impl<'a> std::fmt::Debug for DatumCowTestDisplay<'a> {
+impl std::fmt::Debug for DatumCowTestDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self.0 {
             DatumCow::Borrowed(datum_ref) => {
@@ -164,7 +164,7 @@ impl<'a> std::fmt::Debug for DatumCowTestDisplay<'a> {
 /// More concise display for `ColumnDesc`, to use in tests.
 pub struct ColumnDescTestDisplay<'a>(pub &'a ColumnDesc);
 
-impl<'a> std::fmt::Debug for ColumnDescTestDisplay<'a> {
+impl std::fmt::Debug for ColumnDescTestDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let ColumnDesc {
             data_type,
diff --git a/src/connector/src/connector_common/common.rs b/src/connector/src/connector_common/common.rs
index 1be41b4e09caa..3eaffa93d02a4 100644
--- a/src/connector/src/connector_common/common.rs
+++ b/src/connector/src/connector_common/common.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::BTreeMap;
+use std::hash::Hash;
 use std::io::Write;
 use std::time::Duration;
 
@@ -61,7 +62,7 @@ use aws_types::SdkConfig;
 use risingwave_common::util::env_var::env_var_is_true;
 
 /// A flatten config map for aws auth.
-#[derive(Deserialize, Debug, Clone, WithOptions)]
+#[derive(Deserialize, Debug, Clone, WithOptions, PartialEq)]
 pub struct AwsAuthProps {
     #[serde(rename = "aws.region", alias = "region")]
     pub region: Option<String>,
@@ -161,21 +162,11 @@ impl AwsAuthProps {
 }
 
 #[serde_as]
-#[derive(Debug, Clone, Deserialize, WithOptions)]
-pub struct KafkaCommon {
+#[derive(Debug, Clone, Deserialize, WithOptions, PartialEq, Hash, Eq)]
+pub struct KafkaConnection {
     #[serde(rename = "properties.bootstrap.server", alias = "kafka.brokers")]
     pub brokers: String,
 
-    #[serde(rename = "topic", alias = "kafka.topic")]
-    pub topic: String,
-
-    #[serde(
-        rename = "properties.sync.call.timeout",
-        deserialize_with = "deserialize_duration_from_string",
-        default = "default_kafka_sync_call_timeout"
-    )]
-    pub sync_call_timeout: Duration,
-
     /// Security protocol used for RisingWave to communicate with Kafka brokers. Could be
     /// PLAINTEXT, SSL, SASL_PLAINTEXT or SASL_SSL.
     #[serde(rename = "properties.security.protocol")]
@@ -252,6 +243,20 @@ pub struct KafkaCommon {
 
 #[serde_as]
 #[derive(Debug, Clone, Deserialize, WithOptions)]
+pub struct KafkaCommon {
+    #[serde(rename = "topic", alias = "kafka.topic")]
+    pub topic: String,
+
+    #[serde(
+        rename = "properties.sync.call.timeout",
+        deserialize_with = "deserialize_duration_from_string",
+        default = "default_kafka_sync_call_timeout"
+    )]
+    pub sync_call_timeout: Duration,
+}
+
+#[serde_as]
+#[derive(Debug, Clone, Deserialize, WithOptions, PartialEq)]
 pub struct KafkaPrivateLinkCommon {
     /// This is generated from `private_link_targets` and `private_link_endpoint` in frontend, instead of given by users.
     #[serde(rename = "broker.rewrite.endpoints")]
@@ -269,7 +274,7 @@ pub struct RdKafkaPropertiesCommon {
     /// Maximum Kafka protocol request message size. Due to differing framing overhead between
     /// protocol versions the producer is unable to reliably enforce a strict max message limit at
     /// produce time and may exceed the maximum size by one message in protocol ProduceRequests,
-    /// the broker will enforce the the topic's max.message.bytes limit
+    /// the broker will enforce the topic's max.message.bytes limit
     #[serde(rename = "properties.message.max.bytes")]
     #[serde_as(as = "Option<DisplayFromStr>")]
     pub message_max_bytes: Option<usize>,
@@ -316,7 +321,7 @@ impl RdKafkaPropertiesCommon {
     }
 }
 
-impl KafkaCommon {
+impl KafkaConnection {
     pub(crate) fn set_security_properties(&self, config: &mut ClientConfig) {
         // AWS_MSK_IAM
         if self.is_aws_msk_iam() {
diff --git a/src/connector/src/connector_common/iceberg/mod.rs b/src/connector/src/connector_common/iceberg/mod.rs
index 37d4e5e6f5a08..bf96e474eee80 100644
--- a/src/connector/src/connector_common/iceberg/mod.rs
+++ b/src/connector/src/connector_common/iceberg/mod.rs
@@ -15,7 +15,6 @@
 mod jni_catalog;
 mod mock_catalog;
 mod storage_catalog;
-
 use std::collections::HashMap;
 use std::sync::Arc;
 
@@ -30,6 +29,7 @@ use serde_with::serde_as;
 use url::Url;
 use with_options::WithOptions;
 
+use crate::deserialize_optional_bool_from_string;
 use crate::error::ConnectorResult;
 
 #[serde_as]
@@ -62,6 +62,13 @@ pub struct IcebergCommon {
     /// Full name of table, must include schema name.
     #[serde(rename = "table.name")]
     pub table_name: String,
+
+    #[serde(
+        rename = "s3.path.style.access",
+        default,
+        deserialize_with = "deserialize_optional_bool_from_string"
+    )]
+    pub path_style_access: Option<bool>,
 }
 
 impl IcebergCommon {
@@ -79,7 +86,6 @@ impl IcebergCommon {
     /// For both V1 and V2.
     fn build_jni_catalog_configs(
         &self,
-        path_style_access: &Option<bool>,
         java_catalog_props: &HashMap<String, String>,
     ) -> ConnectorResult<(BaseCatalogConfig, HashMap<String, String>)> {
         let mut iceberg_configs = HashMap::new();
@@ -193,7 +199,7 @@ impl IcebergCommon {
                 self.secret_key.clone().to_string(),
             );
 
-            if let Some(path_style_access) = path_style_access {
+            if let Some(path_style_access) = self.path_style_access {
                 java_catalog_configs.insert(
                     "s3.path-style-access".to_string(),
                     path_style_access.to_string(),
@@ -247,10 +253,7 @@ mod v1 {
             Ok(ret.context("Failed to create table identifier")?)
         }
 
-        fn build_iceberg_configs(
-            &self,
-            path_style_access: &Option<bool>,
-        ) -> ConnectorResult<HashMap<String, String>> {
+        fn build_iceberg_configs(&self) -> ConnectorResult<HashMap<String, String>> {
             let mut iceberg_configs = HashMap::new();
 
             let catalog_type = self.catalog_type().to_string();
@@ -303,7 +306,7 @@ mod v1 {
                 "iceberg.table.io.secret_access_key".to_string(),
                 self.secret_key.clone().to_string(),
             );
-            if let Some(path_style_access) = path_style_access {
+            if let Some(path_style_access) = self.path_style_access {
                 iceberg_configs.insert(
                     "iceberg.table.io.enable_virtual_host_style".to_string(),
                     (!path_style_access).to_string(),
@@ -345,12 +348,11 @@ mod v1 {
         /// TODO: remove the arguments and put them into `IcebergCommon`. Currently the handling in source and sink are different, so pass them separately to be safer.
         pub async fn create_catalog(
             &self,
-            path_style_access: &Option<bool>,
             java_catalog_props: &HashMap<String, String>,
         ) -> ConnectorResult<CatalogRef> {
             match self.catalog_type() {
                 "storage" | "rest" => {
-                    let iceberg_configs = self.build_iceberg_configs(path_style_access)?;
+                    let iceberg_configs = self.build_iceberg_configs()?;
                     let catalog = load_catalog(&iceberg_configs).await?;
                     Ok(catalog)
                 }
@@ -361,7 +363,7 @@ mod v1 {
                 {
                     // Create java catalog
                     let (base_catalog_config, java_catalog_props) =
-                        self.build_jni_catalog_configs(path_style_access, java_catalog_props)?;
+                        self.build_jni_catalog_configs(java_catalog_props)?;
                     let catalog_impl = match catalog_type {
                         "hive" => "org.apache.iceberg.hive.HiveCatalog",
                         "jdbc" => "org.apache.iceberg.jdbc.JdbcCatalog",
@@ -389,11 +391,10 @@ mod v1 {
         /// TODO: remove the arguments and put them into `IcebergCommon`. Currently the handling in source and sink are different, so pass them separately to be safer.
         pub async fn load_table(
             &self,
-            path_style_access: &Option<bool>,
             java_catalog_props: &HashMap<String, String>,
         ) -> ConnectorResult<Table> {
             let catalog = self
-                .create_catalog(path_style_access, java_catalog_props)
+                .create_catalog(java_catalog_props)
                 .await
                 .context("Unable to load iceberg catalog")?;
 
@@ -429,7 +430,6 @@ mod v2 {
         /// TODO: remove the arguments and put them into `IcebergCommon`. Currently the handling in source and sink are different, so pass them separately to be safer.
         pub async fn create_catalog_v2(
             &self,
-            path_style_access: &Option<bool>,
             java_catalog_props: &HashMap<String, String>,
         ) -> ConnectorResult<Arc<dyn CatalogV2>> {
             match self.catalog_type() {
@@ -515,7 +515,7 @@ mod v2 {
                 catalog_type if catalog_type == "hive" || catalog_type == "jdbc" => {
                     // Create java catalog
                     let (base_catalog_config, java_catalog_props) =
-                        self.build_jni_catalog_configs(path_style_access, java_catalog_props)?;
+                        self.build_jni_catalog_configs(java_catalog_props)?;
                     let catalog_impl = match catalog_type {
                         "hive" => "org.apache.iceberg.hive.HiveCatalog",
                         "jdbc" => "org.apache.iceberg.jdbc.JdbcCatalog",
@@ -541,11 +541,10 @@ mod v2 {
         /// TODO: remove the arguments and put them into `IcebergCommon`. Currently the handling in source and sink are different, so pass them separately to be safer.
         pub async fn load_table_v2(
             &self,
-            path_style_access: &Option<bool>,
             java_catalog_props: &HashMap<String, String>,
         ) -> ConnectorResult<TableV2> {
             let catalog = self
-                .create_catalog_v2(path_style_access, java_catalog_props)
+                .create_catalog_v2(java_catalog_props)
                 .await
                 .context("Unable to load iceberg catalog")?;
 
@@ -559,7 +558,6 @@ mod v2 {
         pub async fn load_table_v2_with_metadata(
             &self,
             metadata: TableMetadata,
-            path_style_access: &Option<bool>,
             java_catalog_props: &HashMap<String, String>,
         ) -> ConnectorResult<TableV2> {
             match self.catalog_type() {
@@ -585,10 +583,7 @@ mod v2 {
                         .readonly(true)
                         .build()?)
                 }
-                _ => {
-                    self.load_table_v2(path_style_access, java_catalog_props)
-                        .await
-                }
+                _ => self.load_table_v2(java_catalog_props).await,
             }
         }
     }
diff --git a/src/connector/src/connector_common/iceberg/storage_catalog.rs b/src/connector/src/connector_common/iceberg/storage_catalog.rs
index cd7bb2ca4ba0a..12dfca377ee66 100644
--- a/src/connector/src/connector_common/iceberg/storage_catalog.rs
+++ b/src/connector/src/connector_common/iceberg/storage_catalog.rs
@@ -74,7 +74,7 @@ impl StorageCatalog {
     /// `table_path`: relative path of table dir under warehouse root.
     async fn is_version_hint_exist(&self, table_path: &str) -> Result<bool> {
         self.file_io
-            .is_exist(format!("{table_path}/metadata/version-hint.text").as_str())
+            .exists(format!("{table_path}/metadata/version-hint.text").as_str())
             .await
             .map_err(|err| {
                 Error::new(
@@ -308,7 +308,7 @@ impl Catalog for StorageCatalog {
             }
         };
         let metadata_path = format!("{table_path}/metadata/version-hint.text");
-        self.file_io.is_exist(&metadata_path).await.map_err(|err| {
+        self.file_io.exists(&metadata_path).await.map_err(|err| {
             Error::new(
                 ErrorKind::Unexpected,
                 format!("Failed to check if table exists: {}", err.as_report()),
diff --git a/src/connector/src/connector_common/mod.rs b/src/connector/src/connector_common/mod.rs
index 30d699198ccd4..57b614fdf548a 100644
--- a/src/connector/src/connector_common/mod.rs
+++ b/src/connector/src/connector_common/mod.rs
@@ -19,9 +19,9 @@ pub use mqtt_common::{MqttCommon, QualityOfService as MqttQualityOfService};
 
 mod common;
 pub use common::{
-    AwsAuthProps, AwsPrivateLinkItem, KafkaCommon, KafkaPrivateLinkCommon, KinesisCommon,
-    MongodbCommon, NatsCommon, PulsarCommon, PulsarOauthCommon, RdKafkaPropertiesCommon,
-    PRIVATE_LINK_BROKER_REWRITE_MAP_KEY, PRIVATE_LINK_TARGETS_KEY,
+    AwsAuthProps, AwsPrivateLinkItem, KafkaCommon, KafkaConnection, KafkaPrivateLinkCommon,
+    KinesisCommon, MongodbCommon, NatsCommon, PulsarCommon, PulsarOauthCommon,
+    RdKafkaPropertiesCommon, PRIVATE_LINK_BROKER_REWRITE_MAP_KEY, PRIVATE_LINK_TARGETS_KEY,
 };
 
 mod iceberg;
diff --git a/src/connector/src/error.rs b/src/connector/src/error.rs
index cfee9674d1ce4..c95d7fc97f4e3 100644
--- a/src/connector/src/error.rs
+++ b/src/connector/src/error.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::sync::Arc;
+
 use risingwave_common::array::ArrayError;
 use risingwave_common::error::def_anyhow_newtype;
 use risingwave_pb::PbFieldNotFound;
@@ -29,6 +31,7 @@ def_anyhow_newtype! {
 
     // Common errors
     std::io::Error => transparent,
+    Arc<ConnectorError> => transparent,
 
     // Fine-grained connector errors
     AccessError => transparent,
diff --git a/src/connector/src/parser/additional_columns.rs b/src/connector/src/parser/additional_columns.rs
index 5c91294afa0a7..e5712d95066cb 100644
--- a/src/connector/src/parser/additional_columns.rs
+++ b/src/connector/src/parser/additional_columns.rs
@@ -29,8 +29,8 @@ use risingwave_pb::plan_common::{
 use crate::error::ConnectorResult;
 use crate::source::cdc::MONGODB_CDC_CONNECTOR;
 use crate::source::{
-    AZBLOB_CONNECTOR, GCS_CONNECTOR, KAFKA_CONNECTOR, KINESIS_CONNECTOR, NATS_CONNECTOR,
-    OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR, PULSAR_CONNECTOR,
+    AZBLOB_CONNECTOR, GCS_CONNECTOR, KAFKA_CONNECTOR, KINESIS_CONNECTOR, MQTT_CONNECTOR,
+    NATS_CONNECTOR, OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR, PULSAR_CONNECTOR,
 };
 
 // Hidden additional columns connectors which do not support `include` syntax.
@@ -87,6 +87,7 @@ pub static COMPATIBLE_ADDITIONAL_COLUMNS: LazyLock<HashMap<&'static str, HashSet
                     "collection_name",
                 ]),
             ),
+            (MQTT_CONNECTOR, HashSet::from(["offset", "partition"])),
         ])
     });
 
diff --git a/src/connector/src/parser/common.rs b/src/connector/src/parser/common.rs
index 102014966db42..68779080f0c8c 100644
--- a/src/connector/src/parser/common.rs
+++ b/src/connector/src/parser/common.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use simd_json::prelude::ValueAsContainer;
+use simd_json::prelude::ValueAsObject;
 use simd_json::BorrowedValue;
 
 /// Get a value from a json object by key, case insensitive.
diff --git a/src/connector/src/parser/mod.rs b/src/connector/src/parser/mod.rs
index 2142914aa2503..22cce134906cb 100644
--- a/src/connector/src/parser/mod.rs
+++ b/src/connector/src/parser/mod.rs
@@ -44,7 +44,7 @@ use thiserror_ext::AsReport;
 
 use self::avro::AvroAccessBuilder;
 use self::bytes_parser::BytesAccessBuilder;
-pub use self::mysql::mysql_row_to_owned_row;
+pub use self::mysql::{mysql_datum_to_rw_datum, mysql_row_to_owned_row};
 use self::plain_parser::PlainParser;
 pub use self::postgres::postgres_row_to_owned_row;
 use self::simd_json_parser::DebeziumJsonAccessBuilder;
@@ -215,7 +215,7 @@ impl<'a> MessageMeta<'a> {
     /// Extract the value for the given column.
     ///
     /// Returns `None` if the column is not a meta column.
-    fn value_for_column(self, desc: &SourceColumnDesc) -> Option<DatumRef<'a>> {
+    fn value_for_column(self, desc: &SourceColumnDesc) -> DatumRef<'a> {
         let datum: DatumRef<'_> = match desc.column_type {
             // Row id columns are filled with `NULL` here and will be filled with the real
             // row id generated by `RowIdGenExecutor` later.
@@ -229,11 +229,7 @@ impl<'a> MessageMeta<'a> {
                     KAFKA_TIMESTAMP_COLUMN_NAME,
                     "unexpected kafka meta column name"
                 );
-                kafka_meta.timestamp.map(|ts| {
-                    risingwave_common::cast::i64_to_timestamptz(ts)
-                        .unwrap()
-                        .into()
-                })
+                kafka_meta.extract_timestamp()
             }
             SourceColumnType::Meta if let SourceMeta::DebeziumCdc(cdc_meta) = self.meta => {
                 assert_eq!(
@@ -248,7 +244,7 @@ impl<'a> MessageMeta<'a> {
             SourceColumnType::Meta | SourceColumnType::Normal => return None,
         };
 
-        Some(datum)
+        datum
     }
 }
 
@@ -392,8 +388,7 @@ impl SourceStreamChunkRowWriter<'_> {
                     Ok(A::output_for(
                         self.row_meta
                             .as_ref()
-                            .and_then(|row_meta| row_meta.value_for_column(desc))
-                            .unwrap(), // handled all match cases in internal match, unwrap is safe
+                            .and_then(|row_meta| row_meta.value_for_column(desc)),
                     ))
                 }
                 (&SourceColumnType::Meta, _)
@@ -403,12 +398,11 @@ impl SourceStreamChunkRowWriter<'_> {
                     ) =>
                 {
                     // SourceColumnType is for CDC source only.
-                    return Ok(A::output_for(
+                    Ok(A::output_for(
                         self.row_meta
                             .as_ref()
-                            .and_then(|row_meta| row_meta.value_for_column(desc))
-                            .unwrap(), // handled all match cases in internal match, unwrap is safe
-                    ));
+                            .and_then(|row_meta| row_meta.value_for_column(desc)),
+                    ))
                 }
 
                 (
@@ -434,9 +428,9 @@ impl SourceStreamChunkRowWriter<'_> {
                     }
                 }
                 (_, &Some(AdditionalColumnType::Timestamp(_))) => match self.row_meta {
-                    Some(row_meta) => Ok(A::output_for(
-                        extreact_timestamp_from_meta(row_meta.meta).unwrap_or(None),
-                    )),
+                    Some(row_meta) => {
+                        Ok(A::output_for(extreact_timestamp_from_meta(row_meta.meta)))
+                    }
                     None => parse_field(desc), // parse from payload
                 },
                 (_, &Some(AdditionalColumnType::CollectionName(_))) => {
@@ -445,22 +439,22 @@ impl SourceStreamChunkRowWriter<'_> {
                 }
                 (_, &Some(AdditionalColumnType::Partition(_))) => {
                     // the meta info does not involve spec connector
-                    return Ok(A::output_for(
+                    Ok(A::output_for(
                         self.row_meta
                             .as_ref()
                             .map(|ele| ScalarRefImpl::Utf8(ele.split_id)),
-                    ));
+                    ))
                 }
                 (_, &Some(AdditionalColumnType::Offset(_))) => {
                     // the meta info does not involve spec connector
-                    return Ok(A::output_for(
+                    Ok(A::output_for(
                         self.row_meta
                             .as_ref()
                             .map(|ele| ScalarRefImpl::Utf8(ele.offset)),
-                    ));
+                    ))
                 }
                 (_, &Some(AdditionalColumnType::HeaderInner(ref header_inner))) => {
-                    return Ok(A::output_for(
+                    Ok(A::output_for(
                         self.row_meta
                             .as_ref()
                             .and_then(|ele| {
@@ -473,21 +467,19 @@ impl SourceStreamChunkRowWriter<'_> {
                             .unwrap_or(Datum::None.into()),
                     ))
                 }
-                (_, &Some(AdditionalColumnType::Headers(_))) => {
-                    return Ok(A::output_for(
-                        self.row_meta
-                            .as_ref()
-                            .and_then(|ele| extract_headers_from_meta(ele.meta))
-                            .unwrap_or(None),
-                    ))
-                }
+                (_, &Some(AdditionalColumnType::Headers(_))) => Ok(A::output_for(
+                    self.row_meta
+                        .as_ref()
+                        .and_then(|ele| extract_headers_from_meta(ele.meta))
+                        .unwrap_or(None),
+                )),
                 (_, &Some(AdditionalColumnType::Filename(_))) => {
                     // Filename is used as partition in FS connectors
-                    return Ok(A::output_for(
+                    Ok(A::output_for(
                         self.row_meta
                             .as_ref()
                             .map(|ele| ScalarRefImpl::Utf8(ele.split_id)),
-                    ));
+                    ))
                 }
                 (_, &Some(AdditionalColumnType::Payload(_))) => {
                     // ingest the whole payload as a single column
diff --git a/src/connector/src/parser/mysql.rs b/src/connector/src/parser/mysql.rs
index fe9b77c643de7..9970ad50003c1 100644
--- a/src/connector/src/parser/mysql.rs
+++ b/src/connector/src/parser/mysql.rs
@@ -14,128 +14,209 @@
 
 use std::sync::LazyLock;
 
-use chrono::NaiveDate;
 use mysql_async::Row as MysqlRow;
 use risingwave_common::catalog::Schema;
 use risingwave_common::log::LogSuppresser;
 use risingwave_common::row::OwnedRow;
-use risingwave_common::types::{
-    DataType, Date, Decimal, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz,
-};
-use rust_decimal::Decimal as RustDecimal;
 use thiserror_ext::AsReport;
 
 use crate::parser::util::log_error;
 
 static LOG_SUPPERSSER: LazyLock<LogSuppresser> = LazyLock::new(LogSuppresser::default);
+use anyhow::anyhow;
+use chrono::NaiveDate;
+use risingwave_common::bail;
+use risingwave_common::types::{
+    DataType, Date, Datum, Decimal, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz,
+};
+use rust_decimal::Decimal as RustDecimal;
 
 macro_rules! handle_data_type {
     ($row:expr, $i:expr, $name:expr, $type:ty) => {{
-        let res = $row.take_opt::<Option<$type>, _>($i).unwrap_or(Ok(None));
-        match res {
-            Ok(val) => val.map(|v| ScalarImpl::from(v)),
-            Err(err) => {
-                log_error!($name, err, "parse column failed");
-                None
-            }
+        match $row.take_opt::<Option<$type>, _>($i) {
+            None => bail!("no value found at column: {}, index: {}", $name, $i),
+            Some(Ok(val)) => Ok(val.map(|v| ScalarImpl::from(v))),
+            Some(Err(e)) => Err(anyhow::Error::new(e.clone())
+                .context("failed to deserialize MySQL value into rust value")
+                .context(format!(
+                    "column: {}, index: {}, rust_type: {}",
+                    $name,
+                    $i,
+                    stringify!($type),
+                ))),
         }
     }};
     ($row:expr, $i:expr, $name:expr, $type:ty, $rw_type:ty) => {{
-        let res = $row.take_opt::<Option<$type>, _>($i).unwrap_or(Ok(None));
-        match res {
-            Ok(val) => val.map(|v| ScalarImpl::from(<$rw_type>::from(v))),
-            Err(err) => {
-                log_error!($name, err, "parse column failed");
-                None
-            }
+        match $row.take_opt::<Option<$type>, _>($i) {
+            None => bail!("no value found at column: {}, index: {}", $name, $i),
+            Some(Ok(val)) => Ok(val.map(|v| ScalarImpl::from(<$rw_type>::from(v)))),
+            Some(Err(e)) => Err(anyhow::Error::new(e.clone())
+                .context("failed to deserialize MySQL value into rust value")
+                .context(format!(
+                    "column: {}, index: {}, rust_type: {}",
+                    $name,
+                    $i,
+                    stringify!($ty),
+                ))),
         }
     }};
 }
 
+/// The decoding result can be interpreted as follows:
+/// Ok(value) => The value was found and successfully decoded.
+/// Err(error) => The value was found but could not be decoded,
+///               either because it was not supported,
+///               or there was an error during conversion.
+pub fn mysql_datum_to_rw_datum(
+    mysql_row: &mut MysqlRow,
+    mysql_datum_index: usize,
+    column_name: &str,
+    rw_data_type: &DataType,
+) -> Result<Datum, anyhow::Error> {
+    match rw_data_type {
+        DataType::Boolean => {
+            // TinyInt(1) is used to represent boolean in MySQL
+            // This handles backwards compatibility,
+            // before https://github.com/risingwavelabs/risingwave/pull/19071
+            // we permit boolean and tinyint(1) to be equivalent to boolean in RW.
+            if let Some(Ok(val)) = mysql_row.get_opt::<Option<bool>, _>(mysql_datum_index) {
+                let _ = mysql_row.take::<bool, _>(mysql_datum_index);
+                return Ok(val.map(ScalarImpl::from));
+            }
+            // Bit(1)
+            match mysql_row.take_opt::<Option<Vec<u8>>, _>(mysql_datum_index) {
+                None => bail!(
+                    "no value found at column: {}, index: {}",
+                    column_name,
+                    mysql_datum_index
+                ),
+                Some(Ok(val)) => match val {
+                    None => Ok(None),
+                    Some(val) => match val.as_slice() {
+                        [0] => Ok(Some(ScalarImpl::from(false))),
+                        [1] => Ok(Some(ScalarImpl::from(true))),
+                        _ => Err(anyhow!("invalid value for boolean: {:?}", val)),
+                    },
+                },
+                Some(Err(e)) => Err(anyhow::Error::new(e.clone())
+                    .context("failed to deserialize MySQL value into rust value")
+                    .context(format!(
+                        "column: {}, index: {}, rust_type: Vec<u8>",
+                        column_name, mysql_datum_index,
+                    ))),
+            }
+        }
+        DataType::Int16 => {
+            handle_data_type!(mysql_row, mysql_datum_index, column_name, i16)
+        }
+        DataType::Int32 => {
+            handle_data_type!(mysql_row, mysql_datum_index, column_name, i32)
+        }
+        DataType::Int64 => {
+            handle_data_type!(mysql_row, mysql_datum_index, column_name, i64)
+        }
+        DataType::Float32 => {
+            handle_data_type!(mysql_row, mysql_datum_index, column_name, f32)
+        }
+        DataType::Float64 => {
+            handle_data_type!(mysql_row, mysql_datum_index, column_name, f64)
+        }
+        DataType::Decimal => {
+            handle_data_type!(
+                mysql_row,
+                mysql_datum_index,
+                column_name,
+                RustDecimal,
+                Decimal
+            )
+        }
+        DataType::Varchar => {
+            handle_data_type!(mysql_row, mysql_datum_index, column_name, String)
+        }
+        DataType::Date => {
+            handle_data_type!(mysql_row, mysql_datum_index, column_name, NaiveDate, Date)
+        }
+        DataType::Time => {
+            handle_data_type!(
+                mysql_row,
+                mysql_datum_index,
+                column_name,
+                chrono::NaiveTime,
+                Time
+            )
+        }
+        DataType::Timestamp => {
+            handle_data_type!(
+                mysql_row,
+                mysql_datum_index,
+                column_name,
+                chrono::NaiveDateTime,
+                Timestamp
+            )
+        }
+        DataType::Timestamptz => {
+            match mysql_row.take_opt::<Option<chrono::NaiveDateTime>, _>(mysql_datum_index) {
+                None => bail!(
+                    "no value found at column: {}, index: {}",
+                    column_name,
+                    mysql_datum_index
+                ),
+                Some(Ok(val)) => Ok(val.map(|v| {
+                    ScalarImpl::from(Timestamptz::from_micros(v.and_utc().timestamp_micros()))
+                })),
+                Some(Err(err)) => Err(anyhow::Error::new(err.clone())
+                    .context("failed to deserialize MySQL value into rust value")
+                    .context(format!(
+                        "column: {}, index: {}, rust_type: chrono::NaiveDateTime",
+                        column_name, mysql_datum_index,
+                    ))),
+            }
+        }
+        DataType::Bytea => match mysql_row.take_opt::<Option<Vec<u8>>, _>(mysql_datum_index) {
+            None => bail!(
+                "no value found at column: {}, index: {}",
+                column_name,
+                mysql_datum_index
+            ),
+            Some(Ok(val)) => Ok(val.map(ScalarImpl::from)),
+            Some(Err(err)) => Err(anyhow::Error::new(err.clone())
+                .context("failed to deserialize MySQL value into rust value")
+                .context(format!(
+                    "column: {}, index: {}, rust_type: Vec<u8>",
+                    column_name, mysql_datum_index,
+                ))),
+        },
+        DataType::Jsonb => {
+            handle_data_type!(
+                mysql_row,
+                mysql_datum_index,
+                column_name,
+                serde_json::Value,
+                JsonbVal
+            )
+        }
+        DataType::Interval
+        | DataType::Struct(_)
+        | DataType::List(_)
+        | DataType::Int256
+        | DataType::Serial
+        | DataType::Map(_) => Err(anyhow!(
+            "unsupported data type: {}, set to null",
+            rw_data_type
+        )),
+    }
+}
+
 pub fn mysql_row_to_owned_row(mysql_row: &mut MysqlRow, schema: &Schema) -> OwnedRow {
     let mut datums = vec![];
     for i in 0..schema.fields.len() {
         let rw_field = &schema.fields[i];
         let name = rw_field.name.as_str();
-        let datum = {
-            match rw_field.data_type {
-                DataType::Boolean => {
-                    handle_data_type!(mysql_row, i, name, bool)
-                }
-                DataType::Int16 => {
-                    handle_data_type!(mysql_row, i, name, i16)
-                }
-                DataType::Int32 => {
-                    handle_data_type!(mysql_row, i, name, i32)
-                }
-                DataType::Int64 => {
-                    handle_data_type!(mysql_row, i, name, i64)
-                }
-                DataType::Float32 => {
-                    handle_data_type!(mysql_row, i, name, f32)
-                }
-                DataType::Float64 => {
-                    handle_data_type!(mysql_row, i, name, f64)
-                }
-                DataType::Decimal => {
-                    handle_data_type!(mysql_row, i, name, RustDecimal, Decimal)
-                }
-                DataType::Varchar => {
-                    handle_data_type!(mysql_row, i, name, String)
-                }
-                DataType::Date => {
-                    handle_data_type!(mysql_row, i, name, NaiveDate, Date)
-                }
-                DataType::Time => {
-                    handle_data_type!(mysql_row, i, name, chrono::NaiveTime, Time)
-                }
-                DataType::Timestamp => {
-                    handle_data_type!(mysql_row, i, name, chrono::NaiveDateTime, Timestamp)
-                }
-                DataType::Timestamptz => {
-                    let res = mysql_row
-                        .take_opt::<Option<chrono::NaiveDateTime>, _>(i)
-                        .unwrap_or(Ok(None));
-                    match res {
-                        Ok(val) => val.map(|v| {
-                            ScalarImpl::from(Timestamptz::from_micros(
-                                v.and_utc().timestamp_micros(),
-                            ))
-                        }),
-                        Err(err) => {
-                            log_error!(name, err, "parse column failed");
-                            None
-                        }
-                    }
-                }
-                DataType::Bytea => {
-                    let res = mysql_row
-                        .take_opt::<Option<Vec<u8>>, _>(i)
-                        .unwrap_or(Ok(None));
-                    match res {
-                        Ok(val) => val.map(|v| ScalarImpl::from(v.into_boxed_slice())),
-                        Err(err) => {
-                            log_error!(name, err, "parse column failed");
-                            None
-                        }
-                    }
-                }
-                DataType::Jsonb => {
-                    handle_data_type!(mysql_row, i, name, serde_json::Value, JsonbVal)
-                }
-                DataType::Interval
-                | DataType::Struct(_)
-                | DataType::List(_)
-                | DataType::Int256
-                | DataType::Serial
-                | DataType::Map(_) => {
-                    // Interval, Struct, List, Int256 are not supported
-                    // XXX: is this branch reachable?
-                    if let Ok(suppressed_count) = LOG_SUPPERSSER.check() {
-                        tracing::warn!(column = rw_field.name, ?rw_field.data_type, suppressed_count, "unsupported data type, set to null");
-                    }
-                    None
-                }
+        let datum = match mysql_datum_to_rw_datum(mysql_row, i, name, &rw_field.data_type) {
+            Ok(val) => val,
+            Err(e) => {
+                log_error!(name, e, "parse column failed");
+                None
             }
         };
         datums.push(datum);
diff --git a/src/connector/src/parser/parquet_parser.rs b/src/connector/src/parser/parquet_parser.rs
index db2ace3d2b6dd..4657a518991fc 100644
--- a/src/connector/src/parser/parquet_parser.rs
+++ b/src/connector/src/parser/parquet_parser.rs
@@ -87,11 +87,10 @@ impl ParquetParser {
     /// # Returns
     ///
     /// A `StreamChunk` containing the converted data from the `RecordBatch`.
-
-    // The hidden columns that must be included here are _rw_file and _rw_offset.
-    // Depending on whether the user specifies a primary key (pk), there may be an additional hidden column row_id.
-    // Therefore, the maximum number of hidden columns is three.
-
+    ///
+    /// The hidden columns that must be included here are `_rw_file` and `_rw_offset`.
+    /// Depending on whether the user specifies a primary key (pk), there may be an additional hidden column `row_id`.
+    /// Therefore, the maximum number of hidden columns is three.
     fn convert_record_batch_to_stream_chunk(
         &mut self,
         record_batch: RecordBatch,
diff --git a/src/connector/src/parser/plain_parser.rs b/src/connector/src/parser/plain_parser.rs
index e9c9436fd295f..9a7cb491e697b 100644
--- a/src/connector/src/parser/plain_parser.rs
+++ b/src/connector/src/parser/plain_parser.rs
@@ -163,7 +163,7 @@ impl PlainParser {
             row_op.with_value(self.payload_builder.generate_accessor(data).await?);
         }
 
-        writer.do_insert(|column: &SourceColumnDesc| row_op.access_field(column))?;
+        writer.do_insert(|column: &SourceColumnDesc| row_op.access_field::<false>(column))?;
 
         Ok(ParseResult::Rows)
     }
diff --git a/src/connector/src/parser/unified/json.rs b/src/connector/src/parser/unified/json.rs
index 8ee8f9fe9386f..3584e233d74a1 100644
--- a/src/connector/src/parser/unified/json.rs
+++ b/src/connector/src/parser/unified/json.rs
@@ -28,7 +28,7 @@ use risingwave_common::types::{
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_connector_codec::decoder::utils::extract_decimal;
 use simd_json::prelude::{
-    TypedValue, ValueAsContainer, ValueAsScalar, ValueObjectAccess, ValueTryAsScalar,
+    TypedValue, ValueAsArray, ValueAsScalar, ValueObjectAccess, ValueTryAsScalar,
 };
 use simd_json::{BorrowedValue, ValueType};
 use thiserror_ext::AsReport;
diff --git a/src/connector/src/parser/unified/kv_event.rs b/src/connector/src/parser/unified/kv_event.rs
index 6ab7925b9bb48..f58ed65a54564 100644
--- a/src/connector/src/parser/unified/kv_event.rs
+++ b/src/connector/src/parser/unified/kv_event.rs
@@ -76,13 +76,19 @@ where
         }
     }
 
-    pub fn access_field(&self, desc: &SourceColumnDesc) -> AccessResult<DatumCow<'_>> {
-        match desc.additional_column.column_type {
-            Some(AdditionalColumnType::Key(_)) => self.access_key(&[&desc.name], &desc.data_type),
+    pub fn access_field<const KEY_ONLY: bool>(
+        &self,
+        desc: &SourceColumnDesc,
+    ) -> AccessResult<DatumCow<'_>> {
+        match (&desc.additional_column.column_type, KEY_ONLY) {
+            (Some(AdditionalColumnType::Key(_)), _) => {
+                self.access_key(&[&desc.name], &desc.data_type)
+            }
             // hack here: Get the whole payload as a single column
             // use a special mark empty slice as path to represent the whole payload
-            Some(AdditionalColumnType::Payload(_)) => self.access_value(&[], &desc.data_type),
-            None => self.access_value(&[&desc.name], &desc.data_type),
+            (Some(AdditionalColumnType::Payload(_)), _) => self.access_value(&[], &desc.data_type),
+            (None, false) => self.access_value(&[&desc.name], &desc.data_type),
+            (_, true) => Ok(DatumCow::Owned(None)),
             _ => unreachable!(),
         }
     }
diff --git a/src/connector/src/parser/upsert_parser.rs b/src/connector/src/parser/upsert_parser.rs
index c7bcce9f86a86..54f33ff69ce5c 100644
--- a/src/connector/src/parser/upsert_parser.rs
+++ b/src/connector/src/parser/upsert_parser.rs
@@ -104,10 +104,16 @@ impl UpsertParser {
         } else {
             change_event_op = ChangeEventOperation::Delete;
         }
-        let f = |column: &SourceColumnDesc| row_op.access_field(column);
+
         match change_event_op {
-            ChangeEventOperation::Upsert => writer.do_insert(f)?,
-            ChangeEventOperation::Delete => writer.do_delete(f)?,
+            ChangeEventOperation::Upsert => {
+                let f = |column: &SourceColumnDesc| row_op.access_field::<false>(column);
+                writer.do_insert(f)?
+            }
+            ChangeEventOperation::Delete => {
+                let f = |column: &SourceColumnDesc| row_op.access_field::<true>(column);
+                writer.do_delete(f)?
+            }
         }
         Ok(())
     }
diff --git a/src/connector/src/parser/util.rs b/src/connector/src/parser/util.rs
index 590ba927854d3..ef748ec0e2eb9 100644
--- a/src/connector/src/parser/util.rs
+++ b/src/connector/src/parser/util.rs
@@ -116,10 +116,11 @@ pub(super) async fn bytes_from_url(
     }
 }
 
-pub fn extreact_timestamp_from_meta(meta: &SourceMeta) -> Option<DatumRef<'_>> {
+pub fn extreact_timestamp_from_meta(meta: &SourceMeta) -> DatumRef<'_> {
     match meta {
         SourceMeta::Kafka(kafka_meta) => kafka_meta.extract_timestamp(),
-        SourceMeta::DebeziumCdc(cdc_meta) => Some(cdc_meta.extract_timestamp()),
+        SourceMeta::DebeziumCdc(cdc_meta) => cdc_meta.extract_timestamp(),
+        SourceMeta::Kinesis(kinesis_meta) => kinesis_meta.extract_timestamp(),
         _ => None,
     }
 }
diff --git a/src/connector/src/sink/elasticsearch_opensearch/elasticsearch_converter.rs b/src/connector/src/sink/elasticsearch_opensearch/elasticsearch_converter.rs
index 195c976ba2b76..80dc5e0f1ca6e 100644
--- a/src/connector/src/sink/elasticsearch_opensearch/elasticsearch_converter.rs
+++ b/src/connector/src/sink/elasticsearch_opensearch/elasticsearch_converter.rs
@@ -28,6 +28,7 @@ use super::elasticsearch_opensearch_config::{
 use super::elasticsearch_opensearch_formatter::{BuildBulkPara, ElasticSearchOpenSearchFormatter};
 use crate::sink::Result;
 
+#[expect(clippy::large_enum_variant)]
 pub enum StreamChunkConverter {
     Es(EsStreamChunkConverter),
     Other,
diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs
index bb0a41f63c33d..6b8bcfa4a8c93 100644
--- a/src/connector/src/sink/formatter/mod.rs
+++ b/src/connector/src/sink/formatter/mod.rs
@@ -49,6 +49,7 @@ pub trait SinkFormatter {
     ///   For example append-only without `primary_key` (aka `downstream_pk`) set.
     /// * Value may be None so that messages with same key are removed during log compaction.
     ///   For example debezium tombstone event.
+    #[expect(clippy::type_complexity)]
     fn format_chunk(
         &self,
         chunk: &StreamChunk,
diff --git a/src/connector/src/sink/iceberg/mod.rs b/src/connector/src/sink/iceberg/mod.rs
index d19ea5a3c0f84..e2f00afd5b525 100644
--- a/src/connector/src/sink/iceberg/mod.rs
+++ b/src/connector/src/sink/iceberg/mod.rs
@@ -64,10 +64,7 @@ use crate::connector_common::IcebergCommon;
 use crate::sink::coordinate::CoordinatedSinkWriter;
 use crate::sink::writer::SinkWriter;
 use crate::sink::{Result, SinkCommitCoordinator, SinkParam};
-use crate::{
-    deserialize_bool_from_string, deserialize_optional_bool_from_string,
-    deserialize_optional_string_seq_from_string,
-};
+use crate::{deserialize_bool_from_string, deserialize_optional_string_seq_from_string};
 
 pub const ICEBERG_SINK: &str = "iceberg";
 
@@ -82,13 +79,6 @@ pub struct IcebergConfig {
     #[serde(flatten)]
     common: IcebergCommon,
 
-    #[serde(
-        rename = "s3.path.style.access",
-        default,
-        deserialize_with = "deserialize_optional_bool_from_string"
-    )]
-    pub path_style_access: Option<bool>,
-
     #[serde(
         rename = "primary_key",
         default,
@@ -175,21 +165,21 @@ impl IcebergConfig {
 
     pub async fn create_catalog(&self) -> Result<CatalogRef> {
         self.common
-            .create_catalog(&self.path_style_access, &self.java_catalog_props)
+            .create_catalog(&self.java_catalog_props)
             .await
             .map_err(Into::into)
     }
 
     pub async fn load_table(&self) -> Result<Table> {
         self.common
-            .load_table(&self.path_style_access, &self.java_catalog_props)
+            .load_table(&self.java_catalog_props)
             .await
             .map_err(Into::into)
     }
 
     pub async fn create_catalog_v2(&self) -> Result<Arc<dyn CatalogV2>> {
         self.common
-            .create_catalog_v2(&self.path_style_access, &self.java_catalog_props)
+            .create_catalog_v2(&self.java_catalog_props)
             .await
             .map_err(Into::into)
     }
@@ -1018,10 +1008,10 @@ mod test {
                 catalog_name: Some("demo".to_string()),
                 database_name: Some("demo_db".to_string()),
                 table_name: "demo_table".to_string(),
+                path_style_access: Some(true),
             },
             r#type: "upsert".to_string(),
             force_append_only: false,
-            path_style_access: Some(true),
             primary_key: Some(vec!["v1".to_string()]),
             java_catalog_props: [("jdbc.user", "admin"), ("jdbc.password", "123456")]
                 .into_iter()
diff --git a/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs b/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs
index c5fb3bcc906b5..8a5352772e7c1 100644
--- a/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs
+++ b/src/connector/src/sink/iceberg/prometheus/monitored_partition_writer.rs
@@ -28,6 +28,7 @@ pub struct MonitoredFanoutPartitionedWriterBuilder<B: IcebergWriterBuilder> {
 }
 
 impl<B: IcebergWriterBuilder> MonitoredFanoutPartitionedWriterBuilder<B> {
+    #[expect(dead_code)]
     pub fn new(
         inner: FanoutPartitionedWriterBuilder<B>,
         partition_num: LabelGuardedIntGauge<2>,
diff --git a/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs b/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs
index 634e9ac968f89..90d0bf575d56b 100644
--- a/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs
+++ b/src/connector/src/sink/iceberg/prometheus/monitored_write_writer.rs
@@ -28,6 +28,7 @@ pub struct MonitoredWriteWriterBuilder<B: IcebergWriterBuilder> {
 
 impl<B: IcebergWriterBuilder> MonitoredWriteWriterBuilder<B> {
     /// Create writer context.
+    #[expect(dead_code)]
     pub fn new(
         inner: B,
         write_qps: LabelGuardedIntCounter<2>,
diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs
index 68f20fcf7d581..9fc8da7ef7a48 100644
--- a/src/connector/src/sink/kafka.rs
+++ b/src/connector/src/sink/kafka.rs
@@ -35,7 +35,7 @@ use with_options::WithOptions;
 use super::catalog::{SinkFormat, SinkFormatDesc};
 use super::{Sink, SinkError, SinkParam};
 use crate::connector_common::{
-    AwsAuthProps, KafkaCommon, KafkaPrivateLinkCommon, RdKafkaPropertiesCommon,
+    AwsAuthProps, KafkaCommon, KafkaConnection, KafkaPrivateLinkCommon, RdKafkaPropertiesCommon,
 };
 use crate::sink::formatter::SinkFormatterImpl;
 use crate::sink::log_store::DeliveryFutureManagerAddFuture;
@@ -214,6 +214,9 @@ pub struct KafkaConfig {
     #[serde(flatten)]
     pub common: KafkaCommon,
 
+    #[serde(flatten)]
+    pub connection: KafkaConnection,
+
     #[serde(
         rename = "properties.retry.max",
         default = "_default_max_retries",
@@ -269,6 +272,7 @@ impl From<KafkaConfig> for KafkaProperties {
             time_offset: None,
             upsert: None,
             common: val.common,
+            connection: val.connection,
             rdkafka_properties_common: val.rdkafka_properties_common,
             rdkafka_properties_consumer: Default::default(),
             privatelink_common: val.privatelink_common,
@@ -368,7 +372,7 @@ impl Sink for KafkaSink {
         if !check.check_reachability().await {
             return Err(SinkError::Config(anyhow!(
                 "cannot connect to kafka broker ({})",
-                self.config.common.brokers
+                self.config.connection.brokers
             )));
         }
         Ok(())
@@ -413,11 +417,11 @@ impl KafkaSinkWriter {
             let mut c = ClientConfig::new();
 
             // KafkaConfig configuration
-            config.common.set_security_properties(&mut c);
+            config.connection.set_security_properties(&mut c);
             config.set_client(&mut c);
 
             // ClientConfig configuration
-            c.set("bootstrap.servers", &config.common.brokers);
+            c.set("bootstrap.servers", &config.connection.brokers);
 
             // Create the producer context, will be used to create the producer
             let broker_rewrite_map = config.privatelink_common.broker_rewrite_map.clone();
@@ -426,7 +430,7 @@ impl KafkaSinkWriter {
                 None,
                 None,
                 config.aws_auth_props.clone(),
-                config.common.is_aws_msk_iam(),
+                config.connection.is_aws_msk_iam(),
             )
             .await?;
             let producer_ctx = RwProducerContext::new(ctx_common);
@@ -685,7 +689,7 @@ mod test {
             "broker.rewrite.endpoints".to_string() => "{\"broker1\": \"10.0.0.1:8001\"}".to_string(),
         };
         let config = KafkaConfig::from_btreemap(properties).unwrap();
-        assert_eq!(config.common.brokers, "localhost:9092");
+        assert_eq!(config.connection.brokers, "localhost:9092");
         assert_eq!(config.common.topic, "test");
         assert_eq!(config.max_retry_num, 20);
         assert_eq!(config.retry_interval, Duration::from_millis(500));
diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs
index f6bdffb5fd1e2..03b2fc68476ce 100644
--- a/src/connector/src/sink/mod.rs
+++ b/src/connector/src/sink/mod.rs
@@ -533,6 +533,29 @@ impl SinkMetaClient {
             }
         }
     }
+
+    pub async fn add_sink_fail_evet_log(
+        &self,
+        sink_id: u32,
+        sink_name: String,
+        connector: String,
+        error: String,
+    ) {
+        match self {
+            SinkMetaClient::MetaClient(meta_client) => {
+                match meta_client
+                    .add_sink_fail_evet(sink_id, sink_name, connector, error)
+                    .await
+                {
+                    Ok(_) => {}
+                    Err(e) => {
+                        tracing::warn!(error = %e.as_report(), sink_id = sink_id, "Fialed to add sink fail event to event log.");
+                    }
+                }
+            }
+            SinkMetaClient::MockMetaClient(_) => {}
+        }
+    }
 }
 
 impl SinkWriterParam {
@@ -669,12 +692,10 @@ impl SinkCommitCoordinator for DummySinkCommitCoordinator {
 
 impl SinkImpl {
     pub fn new(mut param: SinkParam) -> Result<Self> {
-        const CONNECTION_NAME_KEY: &str = "connection.name";
         const PRIVATE_LINK_TARGET_KEY: &str = "privatelink.targets";
 
         // remove privatelink related properties if any
         param.properties.remove(PRIVATE_LINK_TARGET_KEY);
-        param.properties.remove(CONNECTION_NAME_KEY);
 
         let sink_type = param
             .properties
diff --git a/src/connector/src/sink/starrocks.rs b/src/connector/src/sink/starrocks.rs
index 456415a62e0d6..35d1033d8ee1b 100644
--- a/src/connector/src/sink/starrocks.rs
+++ b/src/connector/src/sink/starrocks.rs
@@ -371,7 +371,7 @@ pub struct StarrocksSinkWriter {
     pk_indices: Vec<usize>,
     is_append_only: bool,
     client: Option<StarrocksClient>,
-    txn_client: StarrocksTxnClient,
+    txn_client: Arc<StarrocksTxnClient>,
     row_encoder: JsonEncoder,
     executor_id: u64,
     curr_txn_label: Option<String>,
@@ -432,7 +432,7 @@ impl StarrocksSinkWriter {
             pk_indices,
             is_append_only,
             client: None,
-            txn_client: StarrocksTxnClient::new(txn_request_builder),
+            txn_client: Arc::new(StarrocksTxnClient::new(txn_request_builder)),
             row_encoder: JsonEncoder::new_with_starrocks(schema, None),
             executor_id,
             curr_txn_label: None,
@@ -527,6 +527,23 @@ impl StarrocksSinkWriter {
     }
 }
 
+impl Drop for StarrocksSinkWriter {
+    fn drop(&mut self) {
+        if let Some(txn_label) = self.curr_txn_label.take() {
+            let txn_client = self.txn_client.clone();
+            tokio::spawn(async move {
+                if let Err(e) = txn_client.rollback(txn_label.clone()).await {
+                    tracing::error!(
+                        "starrocks rollback transaction error: {:?}, txn label: {}",
+                        e.as_report(),
+                        txn_label
+                    );
+                }
+            });
+        }
+    }
+}
+
 #[async_trait]
 impl SinkWriter for StarrocksSinkWriter {
     type CommitMetadata = Option<SinkMetadata>;
diff --git a/src/connector/src/source/cdc/external/mysql.rs b/src/connector/src/source/cdc/external/mysql.rs
index be0ecf1199291..d58fcf87f957b 100644
--- a/src/connector/src/source/cdc/external/mysql.rs
+++ b/src/connector/src/source/cdc/external/mysql.rs
@@ -121,7 +121,19 @@ impl MySqlExternalTable {
                         DataType::Int16 => Some(ScalarImpl::Int16(val as _)),
                         DataType::Int32 => Some(ScalarImpl::Int32(val as _)),
                         DataType::Int64 => Some(ScalarImpl::Int64(val)),
-                        _ => Err(anyhow!("unexpected default value type for integer column"))?,
+                        DataType::Varchar => {
+                            // should be the Enum type which is mapped to Varchar
+                            Some(ScalarImpl::from(val.to_string()))
+                        }
+                        _ => {
+                            tracing::error!(
+                                column = col_name,
+                                ?data_type,
+                                default_val = val,
+                                "unexpected default value type for column, set default to null"
+                            );
+                            None
+                        }
                     },
                     ColumnDefault::Real(val) => match data_type {
                         DataType::Float32 => Some(ScalarImpl::Float32(F32::from(val as f32))),
@@ -131,7 +143,15 @@ impl MySqlExternalTable {
                                 anyhow!("failed to convert default value to decimal").context(err)
                             })?,
                         )),
-                        _ => Err(anyhow!("unexpected default value type for float column"))?,
+                        _ => {
+                            tracing::error!(
+                                column = col_name,
+                                ?data_type,
+                                default_val = val,
+                                "unexpected default value type for column, set default to null"
+                            );
+                            None
+                        }
                     },
                     ColumnDefault::String(mut val) => {
                         // mysql timestamp is mapped to timestamptz, we use UTC timezone to
diff --git a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
index 1cfc9c1355167..69308a092e2dd 100644
--- a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
+++ b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs
@@ -19,8 +19,12 @@ use async_compression::tokio::bufread::GzipDecoder;
 use async_trait::async_trait;
 use futures::TryStreamExt;
 use futures_async_stream::try_stream;
+use itertools::Itertools;
 use opendal::Operator;
-use parquet::arrow::ParquetRecordBatchStreamBuilder;
+use parquet::arrow::async_reader::AsyncFileReader;
+use parquet::arrow::{parquet_to_arrow_schema, ParquetRecordBatchStreamBuilder, ProjectionMask};
+use parquet::file::metadata::FileMetaData;
+use risingwave_common::array::arrow::IcebergArrowConvert;
 use risingwave_common::array::StreamChunk;
 use risingwave_common::util::tokio_util::compat::FuturesAsyncReadCompatExt;
 use tokio::io::{AsyncRead, BufReader};
@@ -46,6 +50,7 @@ pub struct OpendalReader<Src: OpendalSource> {
     splits: Vec<OpendalFsSplit<Src>>,
     parser_config: ParserConfig,
     source_ctx: SourceContextRef,
+    columns: Option<Vec<Column>>,
 }
 #[async_trait]
 impl<Src: OpendalSource> SplitReader for OpendalReader<Src> {
@@ -57,7 +62,7 @@ impl<Src: OpendalSource> SplitReader for OpendalReader<Src> {
         splits: Vec<OpendalFsSplit<Src>>,
         parser_config: ParserConfig,
         source_ctx: SourceContextRef,
-        _columns: Option<Vec<Column>>,
+        columns: Option<Vec<Column>>,
     ) -> ConnectorResult<Self> {
         let connector = Src::new_enumerator(properties)?;
         let opendal_reader = OpendalReader {
@@ -65,6 +70,7 @@ impl<Src: OpendalSource> SplitReader for OpendalReader<Src> {
             splits,
             parser_config,
             source_ctx,
+            columns,
         };
         Ok(opendal_reader)
     }
@@ -86,7 +92,7 @@ impl<Src: OpendalSource> OpendalReader<Src> {
 
             if let EncodingProperties::Parquet = &self.parser_config.specific.encoding_config {
                 // // If the format is "parquet", use `ParquetParser` to convert `record_batch` into stream chunk.
-                let reader: tokio_util::compat::Compat<opendal::FuturesAsyncReader> = self
+                let mut reader: tokio_util::compat::Compat<opendal::FuturesAsyncReader> = self
                     .connector
                     .op
                     .reader_with(&object_name)
@@ -95,11 +101,19 @@ impl<Src: OpendalSource> OpendalReader<Src> {
                     .into_futures_async_read(..)
                     .await?
                     .compat();
+                let parquet_metadata = reader.get_metadata().await.map_err(anyhow::Error::from)?;
+
+                let file_metadata = parquet_metadata.file_metadata();
+                let column_indices =
+                    extract_valid_column_indices(self.columns.clone(), file_metadata)?;
+                let projection_mask =
+                    ProjectionMask::leaves(file_metadata.schema_descr(), column_indices);
                 // For the Parquet format, we directly convert from a record batch to a stream chunk.
                 // Therefore, the offset of the Parquet file represents the current position in terms of the number of rows read from the file.
                 let record_batch_stream = ParquetRecordBatchStreamBuilder::new(reader)
                     .await?
                     .with_batch_size(self.source_ctx.source_ctrl_opts.chunk_size)
+                    .with_projection(projection_mask)
                     .with_offset(split.offset)
                     .build()?;
 
@@ -215,3 +229,59 @@ impl<Src: OpendalSource> OpendalReader<Src> {
         }
     }
 }
+
+/// Extracts valid column indices from a Parquet file schema based on the user's requested schema.
+///
+/// This function is used for column pruning of Parquet files. It calculates the intersection
+/// between the columns in the currently read Parquet file and the schema provided by the user.
+/// This is useful for reading a `RecordBatch` with the appropriate `ProjectionMask`, ensuring that
+/// only the necessary columns are read.
+///
+/// # Parameters
+/// - `columns`: A vector of `Column` representing the user's requested schema.
+/// - `metadata`: A reference to `FileMetaData` containing the schema and metadata of the Parquet file.
+///
+/// # Returns
+/// - A `ConnectorResult<Vec<usize>>`, which contains the indices of the valid columns in the
+///   Parquet file schema that match the requested schema. If an error occurs during processing,
+///   it returns an appropriate error.
+pub fn extract_valid_column_indices(
+    columns: Option<Vec<Column>>,
+    metadata: &FileMetaData,
+) -> ConnectorResult<Vec<usize>> {
+    match columns {
+        Some(rw_columns) => {
+            let parquet_column_names = metadata
+                .schema_descr()
+                .columns()
+                .iter()
+                .map(|c| c.name())
+                .collect_vec();
+
+            let converted_arrow_schema =
+                parquet_to_arrow_schema(metadata.schema_descr(), metadata.key_value_metadata())
+                    .map_err(anyhow::Error::from)?;
+
+            let valid_column_indices: Vec<usize> = rw_columns
+                .iter()
+                .filter_map(|column| {
+                    parquet_column_names
+                        .iter()
+                        .position(|&name| name == column.name)
+                        .and_then(|pos| {
+                            let arrow_field = IcebergArrowConvert
+                                .to_arrow_field(&column.name, &column.data_type)
+                                .ok()?;
+                            if &arrow_field == converted_arrow_schema.field(pos) {
+                                Some(pos)
+                            } else {
+                                None
+                            }
+                        })
+                })
+                .collect();
+            Ok(valid_column_indices)
+        }
+        None => Ok(vec![]),
+    }
+}
diff --git a/src/connector/src/source/iceberg/mod.rs b/src/connector/src/source/iceberg/mod.rs
index cdeab00649187..60a26e43e1d31 100644
--- a/src/connector/src/source/iceberg/mod.rs
+++ b/src/connector/src/source/iceberg/mod.rs
@@ -65,8 +65,8 @@ impl IcebergProperties {
         if let Some(jdbc_password) = self.jdbc_password.clone() {
             java_catalog_props.insert("jdbc.password".to_string(), jdbc_password);
         }
-        // TODO: support path_style_access and java_catalog_props for iceberg source
-        self.common.load_table_v2(&None, &java_catalog_props).await
+        // TODO: support java_catalog_props for iceberg source
+        self.common.load_table_v2(&java_catalog_props).await
     }
 
     pub async fn load_table_v2_with_metadata(
@@ -82,7 +82,7 @@ impl IcebergProperties {
         }
         // TODO: support path_style_access and java_catalog_props for iceberg source
         self.common
-            .load_table_v2_with_metadata(table_meta, &None, &java_catalog_props)
+            .load_table_v2_with_metadata(table_meta, &java_catalog_props)
             .await
     }
 }
diff --git a/src/connector/src/source/iceberg/parquet_file_reader.rs b/src/connector/src/source/iceberg/parquet_file_reader.rs
index eb98b2fdad215..5a01f2b0ed84d 100644
--- a/src/connector/src/source/iceberg/parquet_file_reader.rs
+++ b/src/connector/src/source/iceberg/parquet_file_reader.rs
@@ -19,7 +19,7 @@ use std::sync::Arc;
 use anyhow::anyhow;
 use bytes::Bytes;
 use futures::future::BoxFuture;
-use futures::TryFutureExt;
+use futures::{FutureExt, TryFutureExt};
 use iceberg::io::{
     FileIOBuilder, FileMetadata, FileRead, S3_ACCESS_KEY_ID, S3_REGION, S3_SECRET_ACCESS_KEY,
 };
@@ -27,9 +27,9 @@ use iceberg::{Error, ErrorKind};
 use opendal::layers::RetryLayer;
 use opendal::services::S3;
 use opendal::Operator;
-use parquet::arrow::async_reader::{AsyncFileReader, MetadataLoader};
+use parquet::arrow::async_reader::AsyncFileReader;
 use parquet::arrow::ParquetRecordBatchStreamBuilder;
-use parquet::file::metadata::ParquetMetaData;
+use parquet::file::metadata::{ParquetMetaData, ParquetMetaDataReader};
 use url::Url;
 
 pub struct ParquetFileReader<R: FileRead> {
@@ -53,12 +53,14 @@ impl<R: FileRead> AsyncFileReader for ParquetFileReader<R> {
     }
 
     fn get_metadata(&mut self) -> BoxFuture<'_, parquet::errors::Result<Arc<ParquetMetaData>>> {
-        Box::pin(async move {
-            let file_size = self.meta.size;
-            let mut loader = MetadataLoader::load(self, file_size as usize, None).await?;
-            loader.load_page_index(false, false).await?;
-            Ok(Arc::new(loader.finish()))
-        })
+        async move {
+            let reader = ParquetMetaDataReader::new();
+            let size = self.meta.size as usize;
+            let meta = reader.load_and_finish(self, size).await?;
+
+            Ok(Arc::new(meta))
+        }
+        .boxed()
     }
 }
 
diff --git a/src/connector/src/source/kafka/enumerator/client.rs b/src/connector/src/source/kafka/enumerator/client.rs
index a425de418ef4a..1d7525bc7a613 100644
--- a/src/connector/src/source/kafka/enumerator/client.rs
+++ b/src/connector/src/source/kafka/enumerator/client.rs
@@ -12,26 +12,34 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
+use std::sync::{Arc, LazyLock, Weak};
 use std::time::Duration;
 
 use anyhow::{anyhow, Context};
 use async_trait::async_trait;
+use moka::future::Cache as MokaCache;
+use moka::ops::compute::Op;
 use prometheus::core::{AtomicI64, GenericGauge};
 use rdkafka::consumer::{BaseConsumer, Consumer};
 use rdkafka::error::KafkaResult;
-use rdkafka::{Offset, TopicPartitionList};
+use rdkafka::{ClientConfig, Offset, TopicPartitionList};
 use risingwave_common::bail;
 use risingwave_common::metrics::LabelGuardedMetric;
 
-use crate::error::ConnectorResult;
+use crate::error::{ConnectorError, ConnectorResult};
 use crate::source::base::SplitEnumerator;
 use crate::source::kafka::split::KafkaSplit;
 use crate::source::kafka::{
-    KafkaContextCommon, KafkaProperties, RwConsumerContext, KAFKA_ISOLATION_LEVEL,
+    KafkaConnection, KafkaContextCommon, KafkaProperties, RwConsumerContext, KAFKA_ISOLATION_LEVEL,
 };
 use crate::source::SourceEnumeratorContextRef;
 
+type KafkaClientType = BaseConsumer<RwConsumerContext>;
+
+pub static SHARED_KAFKA_CLIENT: LazyLock<MokaCache<KafkaConnection, Weak<KafkaClientType>>> =
+    LazyLock::new(|| moka::future::Cache::builder().build());
+
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 pub enum KafkaEnumeratorOffset {
     Earliest,
@@ -44,7 +52,7 @@ pub struct KafkaSplitEnumerator {
     context: SourceEnumeratorContextRef,
     broker_address: String,
     topic: String,
-    client: BaseConsumer<RwConsumerContext>,
+    client: Arc<KafkaClientType>,
     start_offset: KafkaEnumeratorOffset,
 
     // maybe used in the future for batch processing
@@ -68,12 +76,12 @@ impl SplitEnumerator for KafkaSplitEnumerator {
         let mut config = rdkafka::ClientConfig::new();
         let common_props = &properties.common;
 
-        let broker_address = common_props.brokers.clone();
+        let broker_address = properties.connection.brokers.clone();
         let broker_rewrite_map = properties.privatelink_common.broker_rewrite_map.clone();
         let topic = common_props.topic.clone();
         config.set("bootstrap.servers", &broker_address);
         config.set("isolation.level", KAFKA_ISOLATION_LEVEL);
-        common_props.set_security_properties(&mut config);
+        properties.connection.set_security_properties(&mut config);
         properties.set_client(&mut config);
         let mut scan_start_offset = match properties
             .scan_startup_mode
@@ -94,36 +102,64 @@ impl SplitEnumerator for KafkaSplitEnumerator {
             scan_start_offset = KafkaEnumeratorOffset::Timestamp(time_offset)
         }
 
-        // don't need kafka metrics from enumerator
-        let ctx_common = KafkaContextCommon::new(
-            broker_rewrite_map,
-            None,
-            None,
-            properties.aws_auth_props,
-            common_props.is_aws_msk_iam(),
-        )
-        .await?;
-        let client_ctx = RwConsumerContext::new(ctx_common);
-        let client: BaseConsumer<RwConsumerContext> =
-            config.create_with_context(client_ctx).await?;
-
-        // Note that before any SASL/OAUTHBEARER broker connection can succeed the application must call
-        // rd_kafka_oauthbearer_set_token() once – either directly or, more typically, by invoking either
-        // rd_kafka_poll(), rd_kafka_consumer_poll(), rd_kafka_queue_poll(), etc, in order to cause retrieval
-        // of an initial token to occur.
-        // https://docs.confluent.io/platform/current/clients/librdkafka/html/rdkafka_8h.html#a988395722598f63396d7a1bedb22adaf
-        if common_props.is_aws_msk_iam() {
-            #[cfg(not(madsim))]
-            client.poll(Duration::from_secs(10)); // note: this is a blocking call
-            #[cfg(madsim)]
-            client.poll(Duration::from_secs(10)).await;
+        async fn build_kafka_client(
+            config: &ClientConfig,
+            properties: &KafkaProperties,
+            rewrite_map: Option<BTreeMap<String, String>>,
+        ) -> ConnectorResult<KafkaClientType> {
+            let ctx_common = KafkaContextCommon::new(
+                rewrite_map,
+                None,
+                None,
+                properties.aws_auth_props.clone(),
+                properties.connection.is_aws_msk_iam(),
+            )
+            .await?;
+            let client_ctx = RwConsumerContext::new(ctx_common);
+            let client: BaseConsumer<RwConsumerContext> =
+                config.create_with_context(client_ctx).await?;
+
+            // Note that before any SASL/OAUTHBEARER broker connection can succeed the application must call
+            // rd_kafka_oauthbearer_set_token() once – either directly or, more typically, by invoking either
+            // rd_kafka_poll(), rd_kafka_consumer_poll(), rd_kafka_queue_poll(), etc, in order to cause retrieval
+            // of an initial token to occur.
+            // https://docs.confluent.io/platform/current/clients/librdkafka/html/rdkafka_8h.html#a988395722598f63396d7a1bedb22adaf
+            if properties.connection.is_aws_msk_iam() {
+                #[cfg(not(madsim))]
+                client.poll(Duration::from_secs(10)); // note: this is a blocking call
+                #[cfg(madsim)]
+                client.poll(Duration::from_secs(10)).await;
+            }
+            Ok(client)
         }
 
+        let mut client_arc: Option<Arc<KafkaClientType>> = None;
+        SHARED_KAFKA_CLIENT
+            .entry_by_ref(&properties.connection)
+            .and_try_compute_with::<_, _, ConnectorError>(|maybe_entry| async {
+                if let Some(entry) = maybe_entry {
+                    let entry_value = entry.into_value();
+                    if let Some(client) = entry_value.upgrade() {
+                        // return if the client is already built
+                        tracing::info!("reuse existing kafka client for {}", broker_address);
+                        client_arc = Some(client);
+                        return Ok(Op::Nop);
+                    }
+                }
+                let new_client_arc = Arc::new(
+                    build_kafka_client(&config, &properties, broker_rewrite_map.clone()).await?,
+                );
+                tracing::info!("build new kafka client for {}", broker_address);
+                client_arc = Some(new_client_arc.clone());
+                Ok(Op::Put(Arc::downgrade(&new_client_arc)))
+            })
+            .await?;
+
         Ok(Self {
             context,
             broker_address,
             topic,
-            client,
+            client: client_arc.unwrap(),
             start_offset: scan_start_offset,
             stop_offset: KafkaEnumeratorOffset::None,
             sync_call_timeout: properties.common.sync_call_timeout,
@@ -148,7 +184,7 @@ impl SplitEnumerator for KafkaSplitEnumerator {
             .fetch_stop_offset(topic_partitions.as_ref(), &watermarks)
             .await?;
 
-        let ret = topic_partitions
+        let ret: Vec<_> = topic_partitions
             .into_iter()
             .map(|partition| KafkaSplit {
                 topic: self.topic.clone(),
diff --git a/src/connector/src/source/kafka/mod.rs b/src/connector/src/source/kafka/mod.rs
index 125bf73a3529f..030c190eb4942 100644
--- a/src/connector/src/source/kafka/mod.rs
+++ b/src/connector/src/source/kafka/mod.rs
@@ -17,7 +17,7 @@ use std::collections::HashMap;
 use serde::Deserialize;
 use serde_with::{serde_as, DisplayFromStr};
 
-use crate::connector_common::{AwsAuthProps, KafkaPrivateLinkCommon};
+use crate::connector_common::{AwsAuthProps, KafkaConnection, KafkaPrivateLinkCommon};
 
 mod client_context;
 pub mod enumerator;
@@ -143,6 +143,9 @@ pub struct KafkaProperties {
     #[serde(flatten)]
     pub common: KafkaCommon,
 
+    #[serde(flatten)]
+    pub connection: KafkaConnection,
+
     #[serde(flatten)]
     pub rdkafka_properties_common: RdKafkaPropertiesCommon,
 
diff --git a/src/connector/src/source/kafka/private_link.rs b/src/connector/src/source/kafka/private_link.rs
index 6aeebde87b516..3f6f1b8e32da7 100644
--- a/src/connector/src/source/kafka/private_link.rs
+++ b/src/connector/src/source/kafka/private_link.rs
@@ -30,7 +30,6 @@ use crate::error::ConnectorResult;
 use crate::source::kafka::{KAFKA_PROPS_BROKER_KEY, KAFKA_PROPS_BROKER_KEY_ALIAS};
 
 pub const PRIVATELINK_ENDPOINT_KEY: &str = "privatelink.endpoint";
-pub const CONNECTION_NAME_KEY: &str = "connection.name";
 
 #[derive(Debug)]
 pub(super) enum PrivateLinkContextRole {
diff --git a/src/connector/src/source/kafka/source/message.rs b/src/connector/src/source/kafka/source/message.rs
index 247166a156763..38c781deab64e 100644
--- a/src/connector/src/source/kafka/source/message.rs
+++ b/src/connector/src/source/kafka/source/message.rs
@@ -16,7 +16,7 @@ use std::borrow::Cow;
 
 use itertools::Itertools;
 use rdkafka::message::{BorrowedMessage, Headers, OwnedHeaders};
-use rdkafka::Message;
+use rdkafka::{Message, Timestamp};
 use risingwave_common::types::{
     Datum, DatumCow, DatumRef, ListValue, ScalarImpl, ScalarRefImpl, StructValue,
 };
@@ -29,18 +29,15 @@ use crate::source::SourceMeta;
 
 #[derive(Debug, Clone)]
 pub struct KafkaMeta {
-    // timestamp(milliseconds) of message append in mq
-    pub timestamp: Option<i64>,
+    pub timestamp: Timestamp,
     pub headers: Option<OwnedHeaders>,
 }
 
 impl KafkaMeta {
-    pub fn extract_timestamp(&self) -> Option<DatumRef<'_>> {
-        self.timestamp.map(|ts| {
-            Some(ScalarRefImpl::Timestamptz(
-                risingwave_common::cast::i64_to_timestamptz(ts).unwrap(),
-            ))
-        })
+    pub fn extract_timestamp(&self) -> DatumRef<'_> {
+        Some(
+            risingwave_common::types::Timestamptz::from_millis(self.timestamp.to_millis()?)?.into(),
+        )
     }
 
     pub fn extract_header_inner<'a>(
@@ -101,7 +98,7 @@ impl SourceMessage {
             offset: message.offset().to_string(),
             split_id: message.partition().to_string().into(),
             meta: SourceMeta::Kafka(KafkaMeta {
-                timestamp: message.timestamp().to_millis(),
+                timestamp: message.timestamp(),
                 headers: if require_header {
                     message.headers().map(|headers| headers.detach())
                 } else {
diff --git a/src/connector/src/source/kafka/source/reader.rs b/src/connector/src/source/kafka/source/reader.rs
index d58f1b70dd9fc..b9523eca98b57 100644
--- a/src/connector/src/source/kafka/source/reader.rs
+++ b/src/connector/src/source/kafka/source/reader.rs
@@ -64,7 +64,7 @@ impl SplitReader for KafkaSplitReader {
     ) -> Result<Self> {
         let mut config = ClientConfig::new();
 
-        let bootstrap_servers = &properties.common.brokers;
+        let bootstrap_servers = &properties.connection.brokers;
         let broker_rewrite_map = properties.privatelink_common.broker_rewrite_map.clone();
 
         // disable partition eof
@@ -73,7 +73,7 @@ impl SplitReader for KafkaSplitReader {
         config.set("isolation.level", KAFKA_ISOLATION_LEVEL);
         config.set("bootstrap.servers", bootstrap_servers);
 
-        properties.common.set_security_properties(&mut config);
+        properties.connection.set_security_properties(&mut config);
         properties.set_client(&mut config);
 
         let group_id_prefix = properties
@@ -95,7 +95,7 @@ impl SplitReader for KafkaSplitReader {
             // explicitly
             Some(source_ctx.metrics.rdkafka_native_metric.clone()),
             properties.aws_auth_props,
-            properties.common.is_aws_msk_iam(),
+            properties.connection.is_aws_msk_iam(),
         )
         .await?;
 
diff --git a/src/connector/src/source/kinesis/source/message.rs b/src/connector/src/source/kinesis/source/message.rs
index c12679437c1f2..5996096340482 100644
--- a/src/connector/src/source/kinesis/source/message.rs
+++ b/src/connector/src/source/kinesis/source/message.rs
@@ -13,15 +13,24 @@
 // limitations under the License.
 
 use aws_sdk_kinesis::types::Record;
+use aws_smithy_types::DateTime;
 use aws_smithy_types_convert::date_time::DateTimeExt;
+use risingwave_common::types::{DatumRef, ScalarRefImpl};
 
 use crate::source::{SourceMessage, SourceMeta, SplitId};
 
 #[derive(Clone, Debug)]
 pub struct KinesisMeta {
     // from `approximate_arrival_timestamp` of type `Option<aws_smithy_types::DateTime>`
-    #[expect(dead_code)]
-    timestamp: Option<i64>,
+    timestamp: Option<DateTime>,
+}
+
+impl KinesisMeta {
+    pub fn extract_timestamp(&self) -> DatumRef<'_> {
+        Some(ScalarRefImpl::Timestamptz(
+            self.timestamp?.to_chrono_utc().ok()?.into(),
+        ))
+    }
 }
 
 pub fn from_kinesis_record(value: &Record, split_id: SplitId) -> SourceMessage {
@@ -31,9 +40,7 @@ pub fn from_kinesis_record(value: &Record, split_id: SplitId) -> SourceMessage {
         offset: value.sequence_number.clone(),
         split_id,
         meta: SourceMeta::Kinesis(KinesisMeta {
-            timestamp: value
-                .approximate_arrival_timestamp
-                .map(|dt| dt.to_chrono_utc().unwrap().timestamp_millis()),
+            timestamp: value.approximate_arrival_timestamp,
         }),
     }
 }
diff --git a/src/connector/src/source/mqtt/enumerator/mod.rs b/src/connector/src/source/mqtt/enumerator/mod.rs
index 676aba2a55ad8..f2b949fca17b6 100644
--- a/src/connector/src/source/mqtt/enumerator/mod.rs
+++ b/src/connector/src/source/mqtt/enumerator/mod.rs
@@ -17,17 +17,19 @@ use std::sync::atomic::AtomicBool;
 use std::sync::Arc;
 
 use async_trait::async_trait;
+use risingwave_common::bail;
 use rumqttc::v5::{ConnectionError, Event, Incoming};
 use rumqttc::Outgoing;
 use thiserror_ext::AsReport;
 use tokio::sync::RwLock;
 
 use super::source::MqttSplit;
-use super::{MqttError, MqttProperties};
-use crate::error::{ConnectorError, ConnectorResult};
+use super::MqttProperties;
+use crate::error::ConnectorResult;
 use crate::source::{SourceEnumeratorContextRef, SplitEnumerator};
 
 pub struct MqttSplitEnumerator {
+    #[expect(dead_code)]
     topic: String,
     #[expect(dead_code)]
     client: rumqttc::v5::AsyncClient,
@@ -117,10 +119,18 @@ impl SplitEnumerator for MqttSplitEnumerator {
 
     async fn list_splits(&mut self) -> ConnectorResult<Vec<MqttSplit>> {
         if !self.connected.load(std::sync::atomic::Ordering::Relaxed) {
-            return Err(ConnectorError::from(MqttError(format!(
-                "Failed to connect to MQTT broker for topic {}",
-                self.topic
-            ))));
+            let start = std::time::Instant::now();
+            loop {
+                if self.connected.load(std::sync::atomic::Ordering::Relaxed) {
+                    break;
+                }
+
+                if start.elapsed().as_secs() > 10 {
+                    bail!("Failed to connect to mqtt broker");
+                }
+
+                tokio::time::sleep(std::time::Duration::from_millis(500)).await;
+            }
         }
 
         let topics = self.topics.read().await;
diff --git a/src/connector/src/source/mqtt/source/reader.rs b/src/connector/src/source/mqtt/source/reader.rs
index 2c57c8b9966b1..f4ab79bfffed8 100644
--- a/src/connector/src/source/mqtt/source/reader.rs
+++ b/src/connector/src/source/mqtt/source/reader.rs
@@ -52,7 +52,8 @@ impl SplitReader for MqttSplitReader {
     ) -> Result<Self> {
         let (client, eventloop) = properties
             .common
-            .build_client(source_ctx.actor_id, source_ctx.fragment_id as u64)?;
+            .build_client(source_ctx.actor_id, source_ctx.fragment_id as u64)
+            .inspect_err(|e| tracing::error!("Failed to build mqtt client: {}", e.as_report()))?;
 
         let qos = properties.common.qos();
 
diff --git a/src/connector/src/source/nats/mod.rs b/src/connector/src/source/nats/mod.rs
index b8a3b7948aee2..273eac41846ce 100644
--- a/src/connector/src/source/nats/mod.rs
+++ b/src/connector/src/source/nats/mod.rs
@@ -100,7 +100,7 @@ pub struct NatsProperties {
     #[serde(rename = "stream")]
     pub stream: String,
 
-    #[serde(rename = "durable_consumer_name")]
+    #[serde(rename = "consumer.durable_name")]
     pub durable_consumer_name: String,
 
     #[serde(flatten)]
@@ -121,9 +121,6 @@ pub struct NatsPropertiesConsumer {
     #[serde(rename = "consumer.deliver_subject")]
     pub deliver_subject: Option<String>,
 
-    #[serde(rename = "consumer.durable_name")]
-    pub durable_name: Option<String>,
-
     #[serde(rename = "consumer.name")]
     pub name: Option<String>,
 
@@ -217,9 +214,6 @@ impl NatsPropertiesConsumer {
         if let Some(v) = &self.name {
             c.name = Some(v.clone())
         }
-        if let Some(v) = &self.durable_name {
-            c.durable_name = Some(v.clone())
-        }
         if let Some(v) = &self.description {
             c.description = Some(v.clone())
         }
@@ -353,10 +347,7 @@ mod test {
             props.nats_properties_consumer.name,
             Some("foobar".to_string())
         );
-        assert_eq!(
-            props.nats_properties_consumer.durable_name,
-            Some("durable_foobar".to_string())
-        );
+        assert_eq!(props.durable_consumer_name, "durable_foobar".to_string());
         assert_eq!(
             props.nats_properties_consumer.description,
             Some("A description".to_string())
diff --git a/src/connector/src/with_options.rs b/src/connector/src/with_options.rs
index 37cca7ec09bed..6c857de148fad 100644
--- a/src/connector/src/with_options.rs
+++ b/src/connector/src/with_options.rs
@@ -18,6 +18,7 @@ use risingwave_pb::secret::PbSecretRef;
 
 use crate::sink::catalog::SinkFormatDesc;
 use crate::source::cdc::external::CdcTableType;
+use crate::source::cdc::MYSQL_CDC_CONNECTOR;
 use crate::source::iceberg::ICEBERG_CONNECTOR;
 use crate::source::{
     AZBLOB_CONNECTOR, GCS_CONNECTOR, KAFKA_CONNECTOR, OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR,
@@ -104,6 +105,14 @@ pub trait WithPropertiesExt: Get + Sized {
         connector == KAFKA_CONNECTOR
     }
 
+    #[inline(always)]
+    fn is_mysql_cdc_connector(&self) -> bool {
+        let Some(connector) = self.get_connector() else {
+            return false;
+        };
+        connector == MYSQL_CDC_CONNECTOR
+    }
+
     #[inline(always)]
     fn is_cdc_connector(&self) -> bool {
         let Some(connector) = self.get_connector() else {
diff --git a/src/connector/with_options_sink.yaml b/src/connector/with_options_sink.yaml
index ee0984466ff25..3c198b4fadb10 100644
--- a/src/connector/with_options_sink.yaml
+++ b/src/connector/with_options_sink.yaml
@@ -434,11 +434,6 @@ IcebergConfig:
     default: Default::default
 KafkaConfig:
   fields:
-  - name: properties.bootstrap.server
-    field_type: String
-    required: true
-    alias:
-    - kafka.brokers
   - name: topic
     field_type: String
     required: true
@@ -448,6 +443,11 @@ KafkaConfig:
     field_type: Duration
     required: false
     default: 'Duration :: from_secs (5)'
+  - name: properties.bootstrap.server
+    field_type: String
+    required: true
+    alias:
+    - kafka.brokers
   - name: properties.security.protocol
     field_type: String
     comments: |-
@@ -542,7 +542,7 @@ KafkaConfig:
       Maximum Kafka protocol request message size. Due to differing framing overhead between
       protocol versions the producer is unable to reliably enforce a strict max message limit at
       produce time and may exceed the maximum size by one message in protocol ProduceRequests,
-      the broker will enforce the the topic's max.message.bytes limit
+      the broker will enforce the topic's max.message.bytes limit
     required: false
   - name: properties.receive.message.max.bytes
     field_type: usize
diff --git a/src/connector/with_options_source.yaml b/src/connector/with_options_source.yaml
index 20d3fc1714233..41c27a1af7eb6 100644
--- a/src/connector/with_options_source.yaml
+++ b/src/connector/with_options_source.yaml
@@ -117,6 +117,10 @@ IcebergProperties:
     field_type: String
     comments: Full name of table, must include schema name.
     required: true
+  - name: s3.path.style.access
+    field_type: bool
+    required: false
+    default: Default::default
   - name: catalog.jdbc.user
     field_type: String
     required: false
@@ -178,11 +182,6 @@ KafkaProperties:
       combine both key and value fields of the Kafka message.
       TODO: Currently, `Option<bool>` can not be parsed here.
     required: false
-  - name: properties.bootstrap.server
-    field_type: String
-    required: true
-    alias:
-    - kafka.brokers
   - name: topic
     field_type: String
     required: true
@@ -192,6 +191,11 @@ KafkaProperties:
     field_type: Duration
     required: false
     default: 'Duration :: from_secs (5)'
+  - name: properties.bootstrap.server
+    field_type: String
+    required: true
+    alias:
+    - kafka.brokers
   - name: properties.security.protocol
     field_type: String
     comments: |-
@@ -271,7 +275,7 @@ KafkaProperties:
       Maximum Kafka protocol request message size. Due to differing framing overhead between
       protocol versions the producer is unable to reliably enforce a strict max message limit at
       produce time and may exceed the maximum size by one message in protocol ProduceRequests,
-      the broker will enforce the the topic's max.message.bytes limit
+      the broker will enforce the topic's max.message.bytes limit
     required: false
   - name: properties.receive.message.max.bytes
     field_type: usize
@@ -562,9 +566,6 @@ NatsProperties:
   - name: consumer.deliver_subject
     field_type: String
     required: false
-  - name: consumer.durable_name
-    field_type: String
-    required: false
   - name: consumer.name
     field_type: String
     required: false
@@ -643,7 +644,7 @@ NatsProperties:
   - name: stream
     field_type: String
     required: true
-  - name: durable_consumer_name
+  - name: consumer.durable_name
     field_type: String
     required: true
 NexmarkProperties:
diff --git a/src/ctl/src/cmd_impl/hummock/validate_version.rs b/src/ctl/src/cmd_impl/hummock/validate_version.rs
index b6ab7f111aaac..62e988f42f1cf 100644
--- a/src/ctl/src/cmd_impl/hummock/validate_version.rs
+++ b/src/ctl/src/cmd_impl/hummock/validate_version.rs
@@ -206,14 +206,22 @@ pub async fn print_version_delta_in_archive(
 }
 
 fn match_delta(delta: &DeltaType, sst_id: HummockSstableObjectId) -> bool {
-    let DeltaType::IntraLevel(delta) = delta else {
-        return false;
-    };
-    delta
-        .inserted_table_infos
-        .iter()
-        .any(|sst| sst.sst_id == sst_id)
-        || delta.removed_table_ids.iter().any(|sst| *sst == sst_id)
+    match delta {
+        DeltaType::GroupConstruct(_) | DeltaType::GroupDestroy(_) | DeltaType::GroupMerge(_) => {
+            false
+        }
+        DeltaType::IntraLevel(delta) => {
+            delta
+                .inserted_table_infos
+                .iter()
+                .any(|sst| sst.sst_id == sst_id)
+                || delta.removed_table_ids.iter().any(|sst| *sst == sst_id)
+        }
+        DeltaType::NewL0SubLevel(delta) => delta
+            .inserted_table_infos
+            .iter()
+            .any(|sst| sst.sst_id == sst_id),
+    }
 }
 
 fn print_delta(delta: &DeltaType) {
diff --git a/src/error/src/anyhow.rs b/src/error/src/anyhow.rs
index 3d5d6d927fb58..30a46259ffdb5 100644
--- a/src/error/src/anyhow.rs
+++ b/src/error/src/anyhow.rs
@@ -126,7 +126,7 @@ macro_rules! def_anyhow_newtype {
     ) => {
         #[derive(::thiserror::Error, ::std::fmt::Debug)]
         #[error(transparent)]
-        $(#[$attr])* $vis struct $name(#[from] #[backtrace] ::anyhow::Error);
+        $(#[$attr])* $vis struct $name(#[from] #[backtrace] pub ::anyhow::Error);
 
         impl $name {
             /// Unwrap the newtype to get the inner [`anyhow::Error`].
@@ -154,4 +154,30 @@ macro_rules! def_anyhow_newtype {
         )*
     };
 }
-pub use def_anyhow_newtype;
+
+/// Define a newtype + it's variant in the specified type.
+/// This is useful when you want to define a new error type,
+/// but also want to define a variant for it in another enum.
+#[macro_export]
+macro_rules! def_anyhow_variant {
+    (
+        $(#[$attr:meta])* $vis:vis $name:ident,
+        $enum_name:ident $variant_name:ident
+        $(, $from:ty => $context:tt)* $(,)?
+    ) => {
+        def_anyhow_newtype! {
+            $(#[$attr])* $vis $name
+            $(, $from => $context)*
+        }
+
+        $(
+            impl From<$from> for $enum_name {
+                fn from(error: $from) -> Self {
+                    $enum_name::$variant_name($name::from(error))
+                }
+            }
+        )*
+    }
+}
+
+pub use {def_anyhow_newtype, def_anyhow_variant};
diff --git a/src/expr/core/src/window_function/kind.rs b/src/expr/core/src/window_function/kind.rs
index 9abea9b8655a4..e7a16d421483f 100644
--- a/src/expr/core/src/window_function/kind.rs
+++ b/src/expr/core/src/window_function/kind.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use anyhow::Context;
+use enum_as_inner::EnumAsInner;
 use parse_display::{Display, FromStr};
 use risingwave_common::bail;
 
@@ -20,7 +21,8 @@ use crate::aggregate::AggType;
 use crate::Result;
 
 /// Kind of window functions.
-#[derive(Debug, Display, FromStr /* for builtin */, Clone, PartialEq, Eq, Hash)]
+#[expect(clippy::large_enum_variant)]
+#[derive(Debug, Display, FromStr /* for builtin */, Clone, PartialEq, Eq, Hash, EnumAsInner)]
 #[display(style = "snake_case")]
 pub enum WindowFuncKind {
     // General-purpose window functions.
@@ -64,7 +66,7 @@ impl WindowFuncKind {
 }
 
 impl WindowFuncKind {
-    pub fn is_rank(&self) -> bool {
+    pub fn is_numbering(&self) -> bool {
         matches!(self, Self::RowNumber | Self::Rank | Self::DenseRank)
     }
 }
diff --git a/src/expr/core/src/window_function/session.rs b/src/expr/core/src/window_function/session.rs
index 81a77058759bf..cc98c789019cf 100644
--- a/src/expr/core/src/window_function/session.rs
+++ b/src/expr/core/src/window_function/session.rs
@@ -196,7 +196,7 @@ struct SessionFrameGapRef<'a> {
     add_expr: &'a dyn Expression,
 }
 
-impl<'a> SessionFrameGapRef<'a> {
+impl SessionFrameGapRef<'_> {
     fn minimal_next_start_of(&self, end_order_value: impl ToOwnedDatum) -> Datum {
         let row = OwnedRow::new(vec![end_order_value.to_owned_datum()]);
         self.add_expr
diff --git a/src/expr/impl/Cargo.toml b/src/expr/impl/Cargo.toml
index 33c41c09515a5..257cf19b77e7d 100644
--- a/src/expr/impl/Cargo.toml
+++ b/src/expr/impl/Cargo.toml
@@ -38,7 +38,7 @@ chrono = { version = "0.4", default-features = false, features = [
 ] }
 chrono-tz = { version = "0.10", features = ["case-insensitive"] }
 educe = "0.6"
-fancy-regex = "0.13"
+fancy-regex = "0.14"
 futures-async-stream = { workspace = true }
 futures-util = "0.3"
 ginepro = "0.8"
diff --git a/src/expr/impl/src/window_function/aggregate.rs b/src/expr/impl/src/window_function/aggregate.rs
index c173ae8c82995..213de14878627 100644
--- a/src/expr/impl/src/window_function/aggregate.rs
+++ b/src/expr/impl/src/window_function/aggregate.rs
@@ -14,6 +14,7 @@
 
 use std::collections::BTreeSet;
 
+use educe::Educe;
 use futures_util::FutureExt;
 use risingwave_common::array::{DataChunk, Op, StreamChunk};
 use risingwave_common::types::{DataType, Datum};
@@ -22,7 +23,7 @@ use risingwave_common::{bail, must_match};
 use risingwave_common_estimate_size::{EstimateSize, KvSize};
 use risingwave_expr::aggregate::{
     build_append_only, AggCall, AggType, AggregateFunction, AggregateState as AggImplState,
-    BoxedAggregateFunction,
+    BoxedAggregateFunction, PbAggKind,
 };
 use risingwave_expr::sig::FUNCTION_REGISTRY;
 use risingwave_expr::window_function::{
@@ -40,7 +41,6 @@ struct AggregateState<W>
 where
     W: WindowImpl<Key = StateKey, Value = StateValue>,
 {
-    agg_func: BoxedAggregateFunction,
     agg_impl: AggImpl,
     arg_data_types: Vec<DataType>,
     buffer: WindowBuffer<W>,
@@ -65,7 +65,9 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
         direct_args: vec![],
     };
 
-    let (agg_func, agg_impl, enable_delta) = match agg_type {
+    let (agg_impl, enable_delta) = match agg_type {
+        AggType::Builtin(PbAggKind::FirstValue) => (AggImpl::Shortcut(Shortcut::FirstValue), false),
+        AggType::Builtin(PbAggKind::LastValue) => (AggImpl::Shortcut(Shortcut::LastValue), false),
         AggType::Builtin(kind) => {
             let agg_func_sig = FUNCTION_REGISTRY
                 .get(*kind, &arg_data_types, &call.return_type)
@@ -74,28 +76,25 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
             let (agg_impl, enable_delta) =
                 if agg_func_sig.is_retractable() && call.frame.exclusion.is_no_others() {
                     let init_state = agg_func.create_state()?;
-                    (AggImpl::Incremental(init_state), true)
+                    (AggImpl::Incremental(agg_func, init_state), true)
                 } else {
-                    (AggImpl::Full, false)
+                    (AggImpl::Full(agg_func), false)
                 };
-            (agg_func, agg_impl, enable_delta)
+            (agg_impl, enable_delta)
         }
         AggType::UserDefined(_) => {
             // TODO(rc): utilize `retract` method of embedded UDAF to do incremental aggregation
-            let agg_func = build_append_only(&agg_call)?;
-            (agg_func, AggImpl::Full, false)
+            (AggImpl::Full(build_append_only(&agg_call)?), false)
         }
         AggType::WrapScalar(_) => {
             // we have to feed the wrapped scalar function with all the rows in the window,
             // instead of doing incremental aggregation
-            let agg_func = build_append_only(&agg_call)?;
-            (agg_func, AggImpl::Full, false)
+            (AggImpl::Full(build_append_only(&agg_call)?), false)
         }
     };
 
     let this = match &call.frame.bounds {
         FrameBounds::Rows(frame_bounds) => Box::new(AggregateState {
-            agg_func,
             agg_impl,
             arg_data_types,
             buffer: WindowBuffer::<RowsWindow<StateKey, StateValue>>::new(
@@ -106,7 +105,6 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
             buffer_heap_size: KvSize::new(),
         }) as BoxedWindowState,
         FrameBounds::Range(frame_bounds) => Box::new(AggregateState {
-            agg_func,
             agg_impl,
             arg_data_types,
             buffer: WindowBuffer::<RangeWindow<StateValue>>::new(
@@ -117,7 +115,6 @@ pub(super) fn new(call: &WindowFuncCall) -> Result<BoxedWindowState> {
             buffer_heap_size: KvSize::new(),
         }) as BoxedWindowState,
         FrameBounds::Session(frame_bounds) => Box::new(AggregateState {
-            agg_func,
             agg_impl,
             arg_data_types,
             buffer: WindowBuffer::<SessionWindow<StateValue>>::new(
@@ -181,15 +178,31 @@ where
     }
 
     fn slide(&mut self) -> Result<(Datum, StateEvictHint)> {
-        let wrapper = AggregatorWrapper {
-            agg_func: self.agg_func.as_ref(),
-            arg_data_types: &self.arg_data_types,
-        };
         let output = match self.agg_impl {
-            AggImpl::Full => wrapper.aggregate(self.buffer.curr_window_values()),
-            AggImpl::Incremental(ref mut state) => {
+            AggImpl::Full(ref agg_func) => {
+                let wrapper = AggregatorWrapper {
+                    agg_func: agg_func.as_ref(),
+                    arg_data_types: &self.arg_data_types,
+                };
+                wrapper.aggregate(self.buffer.curr_window_values())
+            }
+            AggImpl::Incremental(ref agg_func, ref mut state) => {
+                let wrapper = AggregatorWrapper {
+                    agg_func: agg_func.as_ref(),
+                    arg_data_types: &self.arg_data_types,
+                };
                 wrapper.update(state, self.buffer.consume_curr_window_values_delta())
             }
+            AggImpl::Shortcut(shortcut) => match shortcut {
+                Shortcut::FirstValue => Ok(self
+                    .buffer
+                    .curr_window_first_value()
+                    .and_then(|args| args[0].clone())),
+                Shortcut::LastValue => Ok(self
+                    .buffer
+                    .curr_window_last_value()
+                    .and_then(|args| args[0].clone())),
+            },
         }?;
         let evict_hint = self.slide_inner();
         Ok((output, evict_hint))
@@ -197,16 +210,17 @@ where
 
     fn slide_no_output(&mut self) -> Result<StateEvictHint> {
         match self.agg_impl {
-            AggImpl::Full => {}
-            AggImpl::Incremental(ref mut state) => {
+            AggImpl::Full(..) => {}
+            AggImpl::Incremental(ref agg_func, ref mut state) => {
                 // for incremental agg, we need to update the state even if the caller doesn't need
                 // the output
                 let wrapper = AggregatorWrapper {
-                    agg_func: self.agg_func.as_ref(),
+                    agg_func: agg_func.as_ref(),
                     arg_data_types: &self.arg_data_types,
                 };
                 wrapper.update(state, self.buffer.consume_curr_window_values_delta())?;
             }
+            AggImpl::Shortcut(..) => {}
         };
         Ok(self.slide_inner())
     }
@@ -223,9 +237,18 @@ where
     }
 }
 
+#[derive(Educe)]
+#[educe(Debug)]
 enum AggImpl {
-    Incremental(AggImplState),
-    Full,
+    Incremental(#[educe(Debug(ignore))] BoxedAggregateFunction, AggImplState),
+    Full(#[educe(Debug(ignore))] BoxedAggregateFunction),
+    Shortcut(Shortcut),
+}
+
+#[derive(Debug, Clone, Copy)]
+enum Shortcut {
+    FirstValue,
+    LastValue,
 }
 
 struct AggregatorWrapper<'a> {
diff --git a/src/expr/impl/src/window_function/buffer.rs b/src/expr/impl/src/window_function/buffer.rs
index 57217dda6fd4b..365f3ec4fbc84 100644
--- a/src/expr/impl/src/window_function/buffer.rs
+++ b/src/expr/impl/src/window_function/buffer.rs
@@ -132,6 +132,27 @@ impl<W: WindowImpl> WindowBuffer<W> {
             .map(|Entry { value, .. }| value)
     }
 
+    /// Get the first value in the current window. Time complexity is O(1).
+    pub fn curr_window_first_value(&self) -> Option<&W::Value> {
+        self.curr_window_values().next()
+    }
+
+    /// Get the last value in the current window. Time complexity is O(1).
+    pub fn curr_window_last_value(&self) -> Option<&W::Value> {
+        let (left, right) = self.curr_window_ranges();
+        if !right.is_empty() {
+            self.buffer
+                .get(right.end - 1)
+                .map(|Entry { value, .. }| value)
+        } else if !left.is_empty() {
+            self.buffer
+                .get(left.end - 1)
+                .map(|Entry { value, .. }| value)
+        } else {
+            None
+        }
+    }
+
     /// Consume the delta of values comparing the current window to the previous window.
     /// The delta is not guaranteed to be sorted, especially when frame exclusion is not `NoOthers`.
     pub fn consume_curr_window_values_delta(
diff --git a/src/frontend/Cargo.toml b/src/frontend/Cargo.toml
index 800a90cce5e63..502eaeba2f685 100644
--- a/src/frontend/Cargo.toml
+++ b/src/frontend/Cargo.toml
@@ -32,7 +32,7 @@ easy-ext = "1"
 educe = "0.6"
 either = "1"
 enum-as-inner = "0.6"
-fancy-regex = "0.13.0"
+fancy-regex = "0.14.0"
 fixedbitset = "0.5"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 futures-async-stream = { workspace = true }
@@ -45,6 +45,9 @@ linkme = { version = "0.3", features = ["used_linker"] }
 maplit = "1"
 md5 = "0.7.0"
 memcomparable = "0.2"
+mysql_async = { version = "0.34", default-features = false, features = [
+    "default",
+] }
 num-integer = "0.1"
 parking_lot = { workspace = true }
 parse-display = "0.10"
@@ -78,7 +81,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sha2 = "0.10.7"
 smallvec = { version = "1.13.1", features = ["serde"] }
-speedate = "0.14.0"
+speedate = "0.15.0"
 tempfile = "3"
 thiserror = "1"
 thiserror-ext = { workspace = true }
diff --git a/src/frontend/planner_test/src/lib.rs b/src/frontend/planner_test/src/lib.rs
index 675b99ad0e145..528fa88ef3506 100644
--- a/src/frontend/planner_test/src/lib.rs
+++ b/src/frontend/planner_test/src/lib.rs
@@ -427,7 +427,7 @@ impl TestCase {
                     columns,
                     constraints,
                     if_not_exists,
-                    source_schema,
+                    format_encode,
                     source_watermarks,
                     append_only,
                     on_conflict,
@@ -437,7 +437,7 @@ impl TestCase {
                     wildcard_idx,
                     ..
                 } => {
-                    let source_schema = source_schema.map(|schema| schema.into_v2_with_warning());
+                    let format_encode = format_encode.map(|schema| schema.into_v2_with_warning());
 
                     create_table::handle_create_table(
                         handler_args,
@@ -446,7 +446,7 @@ impl TestCase {
                         wildcard_idx,
                         constraints,
                         if_not_exists,
-                        source_schema,
+                        format_encode,
                         source_watermarks,
                         append_only,
                         on_conflict,
@@ -568,13 +568,13 @@ impl TestCase {
                 Statement::CreateSchema {
                     schema_name,
                     if_not_exists,
-                    user_specified,
+                    owner,
                 } => {
                     create_schema::handle_create_schema(
                         handler_args,
                         schema_name,
                         if_not_exists,
-                        user_specified,
+                        owner,
                     )
                     .await?;
                 }
diff --git a/src/frontend/planner_test/tests/testdata/input/agg.yaml b/src/frontend/planner_test/tests/testdata/input/agg.yaml
index be08720447966..7a9b1b27102c7 100644
--- a/src/frontend/planner_test/tests/testdata/input/agg.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/agg.yaml
@@ -296,20 +296,26 @@
   expected_outputs:
   - logical_plan
   - batch_plan
-- name: distinct on
+- name: distinct on order by distinct columns
   sql: |
     create table t (v1 int, v2 int, v3 int);
     select distinct on (v1, v3) v1, v2 from t order by v1, v3;
   expected_outputs:
   - logical_plan
   - batch_plan
-- name: distinct on
+- name: distinct on order by part of the distinct columns
   sql: |
     create table t (v1 int, v2 int);
     select distinct on (v1) v1, v2 from t order by v1;
   expected_outputs:
   - logical_plan
-- name: distinct on
+- name: distinct on order by distinct columns and additional sorting columns
+  sql: |
+    create table t (id int, ts timestamp, foo int, bar varchar);
+    select distinct on (id) * from t order by id, ts desc;
+  expected_outputs:
+    - logical_plan
+- name: distinct on with expression
   sql: |
     create table t (v1 int, v2 int, v3 int);
     select distinct on(v1) v2 + v3 from t order by v1;
diff --git a/src/frontend/planner_test/tests/testdata/input/explain_json_format.yaml b/src/frontend/planner_test/tests/testdata/input/explain_json_format.yaml
new file mode 100644
index 0000000000000..c3dfe86b1c61a
--- /dev/null
+++ b/src/frontend/planner_test/tests/testdata/input/explain_json_format.yaml
@@ -0,0 +1,18 @@
+- name: test json output format (logical)
+  sql: |
+    CREATE TABLE t (v1 int);
+    explain (logical, format json) SELECT approx_percentile(0.5) WITHIN GROUP (order by v1) from t;
+  expected_outputs:
+    - explain_output
+- name: test json output format (batch)
+  sql: |
+    CREATE TABLE t (v1 int);
+    explain (physical, format json) SELECT approx_percentile(0.5) WITHIN GROUP (order by v1) from t;
+  expected_outputs:
+    - explain_output
+- name: test json output format (stream)
+  sql: |
+    CREATE TABLE t (v1 int);
+    explain (physical, format json) create materialized view m1 as SELECT approx_percentile(0.5) WITHIN GROUP (order by v1) from t;
+  expected_outputs:
+    - explain_output
\ No newline at end of file
diff --git a/src/frontend/planner_test/tests/testdata/input/lateral_subquery.yaml b/src/frontend/planner_test/tests/testdata/input/lateral_subquery.yaml
index 8b9126f18d641..acc07a17d3555 100644
--- a/src/frontend/planner_test/tests/testdata/input/lateral_subquery.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/lateral_subquery.yaml
@@ -134,3 +134,12 @@
     )d on true;
   expected_outputs:
     - batch_plan
+- name: lateral join with CTE
+  sql: |
+    create table t1(x int, y int);
+    create table t2(x int, y int);
+    select * from t1, lateral (
+        with cte as (select * from t2 where t2.y = t1.y)  select x from cte
+    );
+  expected_outputs:
+    - batch_plan
diff --git a/src/frontend/planner_test/tests/testdata/input/nested_loop_temporal_join.yaml b/src/frontend/planner_test/tests/testdata/input/nested_loop_temporal_join.yaml
new file mode 100644
index 0000000000000..404726980dfa8
--- /dev/null
+++ b/src/frontend/planner_test/tests/testdata/input/nested_loop_temporal_join.yaml
@@ -0,0 +1,68 @@
+- name: Left join type for temporal join
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int);
+    select id1, a1, id2, a2 from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 < a2 and b1 > b2
+  expected_outputs:
+  - batch_error
+  - stream_error
+- name: Inner join type for temporal join
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int);
+    select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 > a2 where a2 < 10;
+  expected_outputs:
+  - stream_plan
+- name: Cross join for temporal join
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int);
+    select id1, a1, id2, a2 from stream cross join version FOR SYSTEM_TIME AS OF PROCTIME();
+  expected_outputs:
+  - stream_plan
+- name: implicit join with temporal tables
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select id1, a1, id2, a2 from stream, version FOR SYSTEM_TIME AS OF PROCTIME() where a1 > a2 and a2 < 10;
+  expected_outputs:
+  - stream_plan
+- name: Temporal join with Aggregation
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select count(*) from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 > a2 where a2 < 10;
+  expected_outputs:
+  - stream_plan
+- name: Temporal join type test
+  sql: |
+    create table stream(id1 int, a1 int, b1 int);
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select id1, a1, id2, a2 from stream right join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 < a2;
+  expected_outputs:
+  - stream_error
+- name: Temporal join append only test
+  sql: |
+    create table stream(id1 int, a1 int, b1 int);
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 < a2;
+  expected_outputs:
+  - stream_error
+- name: multi-way temporal join
+  sql: |
+    create table stream(k int, a1 int, b1 int) APPEND ONLY;
+    create table version1(k int, x1 int, y2 int, primary key (k));
+    create table version2(k int, x2 int, y2 int, primary key (k));
+    select stream.k, x1, x2, a1, b1
+    from stream
+    join version1 FOR SYSTEM_TIME AS OF PROCTIME() on stream.a1 < version1.x1
+    join version2 FOR SYSTEM_TIME AS OF PROCTIME() on stream.b1 > version2.y2 where a1 < 10;
+  expected_outputs:
+  - stream_plan
+- name: use CTE as temporal join right table. https://github.com/risingwavelabs/risingwave/issues/18703
+  sql: |
+    create table stream(id1 int, a1 int, b1 int, c1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int, c2 int, primary key (id2));
+    with version as (select * from version) select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on id1 > id2;
+  expected_outputs:
+    - binder_error
\ No newline at end of file
diff --git a/src/frontend/planner_test/tests/testdata/input/shared_source.yml b/src/frontend/planner_test/tests/testdata/input/shared_source.yml
index 71c87ab2e3ce1..4d684422d9fd1 100644
--- a/src/frontend/planner_test/tests/testdata/input/shared_source.yml
+++ b/src/frontend/planner_test/tests/testdata/input/shared_source.yml
@@ -9,7 +9,7 @@
     ) FORMAT PLAIN ENCODE JSON;
   expected_outputs: []
 - with_config_map:
-    enable_shared_source: true
+    streaming_use_shared_source: true
   sql: |
     /* The shared source config doesn't affect table with connector. */
     EXPLAIN CREATE TABLE s(x int,y int)
@@ -25,43 +25,43 @@
 # We use with_config_map to control the config when CREATE SOURCE, and use another SET statement to change the config for CREATE MV
 #
 # batch: All 4 plans should be the same.
-# stream: StreamSourceScan (with backfill) should be used only for the last 2. The first 2 use StreamSource. enable_shared_source changes the behavior of CREATE SOURCE, but not CREATE MATERIALIZED VIEW
+# stream: StreamSourceScan (with backfill) should be used only for the last 2. The first 2 use StreamSource. streaming_use_shared_source changes the behavior of CREATE SOURCE, but not CREATE MATERIALIZED VIEW
 - with_config_map:
-    enable_shared_source: false
+    streaming_use_shared_source: false
   before:
     - create_source
   sql: |
-    SET enable_shared_source = false;
+    SET streaming_use_shared_source = false;
     select * from s;
   expected_outputs:
     - batch_plan
     - stream_plan
 - with_config_map:
-    enable_shared_source: false
+    streaming_use_shared_source: false
   before:
     - create_source
   sql: |
-    SET enable_shared_source = true;
+    SET streaming_use_shared_source = true;
     select * from s;
   expected_outputs:
     - batch_plan
     - stream_plan
 - with_config_map:
-    enable_shared_source: true
+    streaming_use_shared_source: true
   before:
     - create_source
   sql: |
-    SET enable_shared_source = false;
+    SET streaming_use_shared_source = false;
     select * from s;
   expected_outputs:
     - batch_plan
     - stream_plan
 - with_config_map:
-    enable_shared_source: true
+    streaming_use_shared_source: true
   before:
     - create_source
   sql: |
-    SET enable_shared_source = true;
+    SET streaming_use_shared_source = true;
     select * from s;
   expected_outputs:
     - batch_plan
diff --git a/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml b/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml
index f35703128ae7d..7194a2806a2e8 100644
--- a/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml
+++ b/src/frontend/planner_test/tests/testdata/input/subquery_expr_correlated.yaml
@@ -554,3 +554,41 @@
   expected_outputs:
     - batch_plan
     - stream_plan
+- name: improve join transpose rule to handle join with output indices
+  sql: |
+    with rawdata as (
+      select 'first' as w, '{"x":{"value":123},"y":{"value":[1,2,3]},"z":{"value":[{"a":4,"b":5},{"a":6,"b":7}]}}'::jsonb as rawjson
+      union all
+      select 'second', '{"x":{"value":456},"y":{"value":[7,8,9]},"z":{"value":[{"a":0,"b":1},{"a":2,"b":3}]}}'::jsonb
+    )
+    select
+      array(
+        select
+          array(
+            select 1
+            from jsonb_array_elements(
+              case
+                when jsonb_typeof(o.value->'value') = 'array' then o.value->'value'
+                else '[]'::jsonb
+              end
+            ) with ordinality as x(v, i), jsonb_each (
+              case
+                when jsonb_typeof(v) = 'object' then v
+                else '{}'::jsonb
+              end
+            ) as j
+        )
+        from jsonb_each(rawjson) AS o
+        where jsonb_typeof(o.value) = 'object' and o.value ? 'value'
+      ) as kv
+    from rawdata
+  expected_outputs:
+    - optimized_logical_plan_for_batch
+- name: subquery with CTE
+  sql: |
+    create table t1(x int, y int);
+    create table t2(x int, y int);
+    select * from t1 where t1.x = ( with cte as (select * from t2 where t2.y = t1.y)  select x from cte limit 1);
+  expected_outputs:
+    - batch_plan
+    - stream_plan
\ No newline at end of file
diff --git a/src/frontend/planner_test/tests/testdata/output/agg.yaml b/src/frontend/planner_test/tests/testdata/output/agg.yaml
index 197bca883ca77..3f373d7181166 100644
--- a/src/frontend/planner_test/tests/testdata/output/agg.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/agg.yaml
@@ -492,7 +492,7 @@
         └─BatchGroupTopN { order: [t.v3 ASC, t.v1 ASC], limit: 1, offset: 0, group_key: [t.v1, t.v3] }
           └─BatchExchange { order: [], dist: HashShard(t.v1, t.v3) }
             └─BatchScan { table: t, columns: [t.v1, t.v2, t.v3], distribution: SomeShard }
-- name: distinct on
+- name: distinct on order by distinct columns
   sql: |
     create table t (v1 int, v2 int, v3 int);
     select distinct on (v1, v3) v1, v2 from t order by v1, v3;
@@ -508,7 +508,7 @@
         └─BatchGroupTopN { order: [t.v1 ASC, t.v3 ASC], limit: 1, offset: 0, group_key: [t.v1, t.v3] }
           └─BatchExchange { order: [], dist: HashShard(t.v1, t.v3) }
             └─BatchScan { table: t, columns: [t.v1, t.v2, t.v3], distribution: SomeShard }
-- name: distinct on
+- name: distinct on order by part of the distinct columns
   sql: |
     create table t (v1 int, v2 int);
     select distinct on (v1) v1, v2 from t order by v1;
@@ -516,7 +516,15 @@
     LogicalTopN { order: [t.v1 ASC], limit: 1, offset: 0, group_key: [t.v1] }
     └─LogicalProject { exprs: [t.v1, t.v2] }
       └─LogicalScan { table: t, columns: [t.v1, t.v2, t._row_id] }
-- name: distinct on
+- name: distinct on order by distinct columns and additional sorting columns
+  sql: |
+    create table t (id int, ts timestamp, foo int, bar varchar);
+    select distinct on (id) * from t order by id, ts desc;
+  logical_plan: |-
+    LogicalTopN { order: [t.id ASC, t.ts DESC], limit: 1, offset: 0, group_key: [t.id] }
+    └─LogicalProject { exprs: [t.id, t.ts, t.foo, t.bar] }
+      └─LogicalScan { table: t, columns: [t.id, t.ts, t.foo, t.bar, t._row_id] }
+- name: distinct on with expression
   sql: |
     create table t (v1 int, v2 int, v3 int);
     select distinct on(v1) v2 + v3 from t order by v1;
diff --git a/src/frontend/planner_test/tests/testdata/output/explain_json_format.yaml b/src/frontend/planner_test/tests/testdata/output/explain_json_format.yaml
new file mode 100644
index 0000000000000..9017c0609c253
--- /dev/null
+++ b/src/frontend/planner_test/tests/testdata/output/explain_json_format.yaml
@@ -0,0 +1,147 @@
+# This file is automatically generated. See `src/frontend/planner_test/README.md` for more information.
+- name: test json output format (logical)
+  sql: |
+    CREATE TABLE t (v1 int);
+    explain (logical, format json) SELECT approx_percentile(0.5) WITHIN GROUP (order by v1) from t;
+  explain_output: |
+    {
+      "name": "LogicalAgg",
+      "fields": {
+        "aggs": [
+          "approx_percentile($expr1)"
+        ]
+      },
+      "children": [
+        {
+          "name": "LogicalProject",
+          "fields": {
+            "exprs": [
+              "t.v1::Float64 as $expr1"
+            ]
+          },
+          "children": [
+            {
+              "name": "LogicalScan",
+              "fields": {
+                "columns": [
+                  "v1"
+                ],
+                "table": "t"
+              },
+              "children": []
+            }
+          ]
+        }
+      ]
+    }
+- name: test json output format (batch)
+  sql: |
+    CREATE TABLE t (v1 int);
+    explain (physical, format json) SELECT approx_percentile(0.5) WITHIN GROUP (order by v1) from t;
+  explain_output: |
+    {
+      "name": "BatchSimpleAgg",
+      "fields": {
+        "aggs": [
+          "approx_percentile($expr1)"
+        ]
+      },
+      "children": [
+        {
+          "name": "BatchExchange",
+          "fields": {
+            "dist": "Single",
+            "order": []
+          },
+          "children": [
+            {
+              "name": "BatchProject",
+              "fields": {
+                "exprs": [
+                  "t.v1::Float64 as $expr1"
+                ]
+              },
+              "children": [
+                {
+                  "name": "BatchScan",
+                  "fields": {
+                    "columns": [
+                      "v1"
+                    ],
+                    "table": "t"
+                  },
+                  "children": []
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+- name: test json output format (stream)
+  sql: |
+    CREATE TABLE t (v1 int);
+    explain (physical, format json) create materialized view m1 as SELECT approx_percentile(0.5) WITHIN GROUP (order by v1) from t;
+  explain_output: |
+    {
+      "name": "StreamMaterialize",
+      "fields": {
+        "columns": [
+          "approx_percentile"
+        ],
+        "pk_columns": [],
+        "pk_conflict": "NoCheck",
+        "stream_key": []
+      },
+      "children": [
+        {
+          "name": "StreamGlobalApproxPercentile",
+          "fields": {
+            "quantile": "0.5:Float64",
+            "relative_error": "0.01:Float64"
+          },
+          "children": [
+            {
+              "name": "StreamExchange",
+              "fields": {
+                "dist": "Single"
+              },
+              "children": [
+                {
+                  "name": "StreamLocalApproxPercentile",
+                  "fields": {
+                    "percentile_col": "$expr1",
+                    "quantile": "0.5:Float64",
+                    "relative_error": "0.01:Float64"
+                  },
+                  "children": [
+                    {
+                      "name": "StreamProject",
+                      "fields": {
+                        "exprs": [
+                          "t.v1::Float64 as $expr1",
+                          "t._row_id"
+                        ]
+                      },
+                      "children": [
+                        {
+                          "name": "StreamTableScan",
+                          "fields": {
+                            "columns": [
+                              "v1",
+                              "_row_id"
+                            ],
+                            "table": "t"
+                          },
+                          "children": []
+                        }
+                      ]
+                    }
+                  ]
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
diff --git a/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml b/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml
index 815890d6a73b8..2e3046c7a2e4b 100644
--- a/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/lateral_subquery.yaml
@@ -270,3 +270,18 @@
               │     └─BatchValues { rows: [[1:Int32, '2024-06-20 19:01:00+00:00':Timestamptz]] }
               └─BatchExchange { order: [], dist: HashShard(r.src_id) }
                 └─BatchScan { table: r, columns: [r.ts, r.src_id, r.dev_id], distribution: SomeShard }
+- name: lateral join with CTE
+  sql: |
+    create table t1(x int, y int);
+    create table t2(x int, y int);
+    select * from t1, lateral (
+        with cte as (select * from t2 where t2.y = t1.y)  select x from cte
+    );
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchHashJoin { type: Inner, predicate: t1.y IS NOT DISTINCT FROM t2.y, output: [t1.x, t1.y, t2.x] }
+      ├─BatchExchange { order: [], dist: HashShard(t1.y) }
+      │ └─BatchScan { table: t1, columns: [t1.x, t1.y], distribution: SomeShard }
+      └─BatchExchange { order: [], dist: HashShard(t2.y) }
+        └─BatchFilter { predicate: IsNotNull(t2.y) }
+          └─BatchScan { table: t2, columns: [t2.x, t2.y], distribution: SomeShard }
diff --git a/src/frontend/planner_test/tests/testdata/output/nested_loop_temporal_join.yaml b/src/frontend/planner_test/tests/testdata/output/nested_loop_temporal_join.yaml
new file mode 100644
index 0000000000000..258c33f6f504e
--- /dev/null
+++ b/src/frontend/planner_test/tests/testdata/output/nested_loop_temporal_join.yaml
@@ -0,0 +1,112 @@
+# This file is automatically generated. See `src/frontend/planner_test/README.md` for more information.
+- name: Left join type for temporal join
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int);
+    select id1, a1, id2, a2 from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 < a2 and b1 > b2
+  batch_error: |-
+    Not supported: do not support temporal join for batch queries
+    HINT: please use temporal join in streaming queries
+  stream_error: |-
+    Not supported: Temporal join requires an inner join
+    HINT: Please use an inner join
+- name: Inner join type for temporal join
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int);
+    select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 > a2 where a2 < 10;
+  stream_plan: |-
+    StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), version._row_id(hidden)], stream_key: [stream._row_id, version._row_id], pk_columns: [stream._row_id, version._row_id], pk_conflict: NoCheck }
+    └─StreamExchange { dist: HashShard(stream._row_id, version._row_id) }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate:  AND (stream.a1 > version.a2) AND (version.a2 < 10:Int32), nested_loop: true, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id, version._row_id] }
+        ├─StreamExchange { dist: Broadcast }
+        │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
+        └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version._row_id) }
+          └─StreamTableScan { table: version, columns: [version.id2, version.a2, version._row_id], stream_scan_type: UpstreamOnly, stream_key: [version._row_id], pk: [_row_id], dist: UpstreamHashShard(version._row_id) }
+- name: Cross join for temporal join
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int);
+    select id1, a1, id2, a2 from stream cross join version FOR SYSTEM_TIME AS OF PROCTIME();
+  stream_plan: |-
+    StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), version._row_id(hidden)], stream_key: [stream._row_id, version._row_id], pk_columns: [stream._row_id, version._row_id], pk_conflict: NoCheck }
+    └─StreamExchange { dist: HashShard(stream._row_id, version._row_id) }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: , nested_loop: true, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id, version._row_id] }
+        ├─StreamExchange { dist: Broadcast }
+        │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
+        └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version._row_id) }
+          └─StreamTableScan { table: version, columns: [version.id2, version.a2, version._row_id], stream_scan_type: UpstreamOnly, stream_key: [version._row_id], pk: [_row_id], dist: UpstreamHashShard(version._row_id) }
+- name: implicit join with temporal tables
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select id1, a1, id2, a2 from stream, version FOR SYSTEM_TIME AS OF PROCTIME() where a1 > a2 and a2 < 10;
+  stream_plan: |-
+    StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden)], stream_key: [stream._row_id, id2], pk_columns: [stream._row_id, id2], pk_conflict: NoCheck }
+    └─StreamExchange { dist: HashShard(version.id2, stream._row_id) }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate:  AND (stream.a1 > version.a2) AND (version.a2 < 10:Int32), nested_loop: true, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
+        ├─StreamExchange { dist: Broadcast }
+        │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
+        └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
+          └─StreamTableScan { table: version, columns: [version.id2, version.a2], stream_scan_type: UpstreamOnly, stream_key: [version.id2], pk: [id2], dist: UpstreamHashShard(version.id2) }
+- name: Temporal join with Aggregation
+  sql: |
+    create table stream(id1 int, a1 int, b1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select count(*) from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 > a2 where a2 < 10;
+  stream_plan: |-
+    StreamMaterialize { columns: [count], stream_key: [], pk_columns: [], pk_conflict: NoCheck }
+    └─StreamProject { exprs: [sum0(count)] }
+      └─StreamSimpleAgg [append_only] { aggs: [sum0(count), count] }
+        └─StreamExchange { dist: Single }
+          └─StreamStatelessSimpleAgg { aggs: [count] }
+            └─StreamTemporalJoin { type: Inner, append_only: true, predicate:  AND (stream.a1 > version.a2) AND (version.a2 < 10:Int32), nested_loop: true, output: [stream._row_id, version.id2] }
+              ├─StreamExchange { dist: Broadcast }
+              │ └─StreamTableScan { table: stream, columns: [stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
+              └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
+                └─StreamTableScan { table: version, columns: [version.a2, version.id2], stream_scan_type: UpstreamOnly, stream_key: [version.id2], pk: [id2], dist: UpstreamHashShard(version.id2) }
+- name: Temporal join type test
+  sql: |
+    create table stream(id1 int, a1 int, b1 int);
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select id1, a1, id2, a2 from stream right join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 < a2;
+  stream_error: |-
+    Not supported: streaming nested-loop join
+    HINT: The non-equal join in the query requires a nested-loop join executor, which could be very expensive to run. Consider rewriting the query to use dynamic filter as a substitute if possible.
+    See also: https://docs.risingwave.com/docs/current/sql-pattern-dynamic-filters/
+- name: Temporal join append only test
+  sql: |
+    create table stream(id1 int, a1 int, b1 int);
+    create table version(id2 int, a2 int, b2 int, primary key (id2));
+    select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 < a2;
+  stream_error: |-
+    Not supported: Nested-loop Temporal join requires the left hash side to be append only
+    HINT: Please ensure the left hash side is append only
+- name: multi-way temporal join
+  sql: |
+    create table stream(k int, a1 int, b1 int) APPEND ONLY;
+    create table version1(k int, x1 int, y2 int, primary key (k));
+    create table version2(k int, x2 int, y2 int, primary key (k));
+    select stream.k, x1, x2, a1, b1
+    from stream
+    join version1 FOR SYSTEM_TIME AS OF PROCTIME() on stream.a1 < version1.x1
+    join version2 FOR SYSTEM_TIME AS OF PROCTIME() on stream.b1 > version2.y2 where a1 < 10;
+  stream_plan: |-
+    StreamMaterialize { columns: [k, x1, x2, a1, b1, stream._row_id(hidden), version1.k(hidden), version2.k(hidden)], stream_key: [stream._row_id, version1.k, version2.k], pk_columns: [stream._row_id, version1.k, version2.k], pk_conflict: NoCheck }
+    └─StreamExchange { dist: HashShard(stream._row_id, version1.k, version2.k) }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate:  AND (stream.b1 > version2.y2), nested_loop: true, output: [stream.k, version1.x1, version2.x2, stream.a1, stream.b1, stream._row_id, version1.k, version2.k] }
+        ├─StreamExchange { dist: Broadcast }
+        │ └─StreamTemporalJoin { type: Inner, append_only: true, predicate:  AND (stream.a1 < version1.x1), nested_loop: true, output: [stream.k, stream.a1, stream.b1, version1.x1, stream._row_id, version1.k] }
+        │   ├─StreamExchange { dist: Broadcast }
+        │   │ └─StreamFilter { predicate: (stream.a1 < 10:Int32) }
+        │   │   └─StreamTableScan { table: stream, columns: [stream.k, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
+        │   └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version1.k) }
+        │     └─StreamTableScan { table: version1, columns: [version1.x1, version1.k], stream_scan_type: UpstreamOnly, stream_key: [version1.k], pk: [k], dist: UpstreamHashShard(version1.k) }
+        └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version2.k) }
+          └─StreamTableScan { table: version2, columns: [version2.x2, version2.y2, version2.k], stream_scan_type: UpstreamOnly, stream_key: [version2.k], pk: [k], dist: UpstreamHashShard(version2.k) }
+- name: use CTE as temporal join right table. https://github.com/risingwavelabs/risingwave/issues/18703
+  sql: |
+    create table stream(id1 int, a1 int, b1 int, c1 int) APPEND ONLY;
+    create table version(id2 int, a2 int, b2 int, c2 int, primary key (id2));
+    with version as (select * from version) select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on id1 > id2;
+  binder_error: 'Bind error: Right table of a temporal join should not be a CTE. It should be a table, index, or materialized view'
diff --git a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml
index d6b90da0a8c1a..1f667a63d2fde 100644
--- a/src/frontend/planner_test/tests/testdata/output/nexmark.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/nexmark.yaml
@@ -998,7 +998,7 @@
     ON mod(B.auction, 10000) = S.key
   sink_plan: |-
     StreamSink { type: append-only, columns: [auction, bidder, price, date_time, value, bid._row_id(hidden), $expr10018(hidden), side_input.key(hidden)] }
-    └─StreamTemporalJoin { type: Inner, append_only: true, predicate: $expr1 = side_input.key, output: [bid.auction, bid.bidder, bid.price, bid.date_time, side_input.value, bid._row_id, $expr1, side_input.key] }
+    └─StreamTemporalJoin { type: Inner, append_only: true, predicate: $expr1 = side_input.key, nested_loop: false, output: [bid.auction, bid.bidder, bid.price, bid.date_time, side_input.value, bid._row_id, $expr1, side_input.key] }
       ├─StreamExchange { dist: HashShard($expr1) }
       │ └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.date_time, (bid.auction % 10000:Int32) as $expr1, bid._row_id] }
       │   └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
@@ -1007,7 +1007,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [auction, bidder, price, date_time, value, bid._row_id(hidden), $expr1(hidden), side_input.key(hidden)], stream_key: [bid._row_id, $expr1], pk_columns: [bid._row_id, $expr1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(bid._row_id, $expr1) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: $expr1 = side_input.key, output: [bid.auction, bid.bidder, bid.price, bid.date_time, side_input.value, bid._row_id, $expr1, side_input.key] }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: $expr1 = side_input.key, nested_loop: false, output: [bid.auction, bid.bidder, bid.price, bid.date_time, side_input.value, bid._row_id, $expr1, side_input.key] }
         ├─StreamExchange { dist: HashShard($expr1) }
         │ └─StreamProject { exprs: [bid.auction, bid.bidder, bid.price, bid.date_time, (bid.auction % 10000:Int32) as $expr1, bid._row_id] }
         │   └─StreamTableScan { table: bid, columns: [bid.auction, bid.bidder, bid.price, bid.date_time, bid._row_id], stream_scan_type: ArrangementBackfill, stream_key: [bid._row_id], pk: [_row_id], dist: UpstreamHashShard(bid._row_id) }
@@ -1020,7 +1020,7 @@
     └── StreamExchange Hash([5, 6]) from 1
 
     Fragment 1
-    StreamTemporalJoin { type: Inner, append_only: true, predicate: $expr1 = side_input.key, output: [bid.auction, bid.bidder, bid.price, bid.date_time, side_input.value, bid._row_id, $expr1, side_input.key] }
+    StreamTemporalJoin { type: Inner, append_only: true, predicate: $expr1 = side_input.key, nested_loop: false, output: [bid.auction, bid.bidder, bid.price, bid.date_time, side_input.value, bid._row_id, $expr1, side_input.key] }
     ├── StreamExchange Hash([4]) from 2
     └── StreamExchange NoShuffle from 3
 
diff --git a/src/frontend/planner_test/tests/testdata/output/shared_source.yml b/src/frontend/planner_test/tests/testdata/output/shared_source.yml
index 5083c23952f3b..83fde26bfc7d2 100644
--- a/src/frontend/planner_test/tests/testdata/output/shared_source.yml
+++ b/src/frontend/planner_test/tests/testdata/output/shared_source.yml
@@ -27,11 +27,11 @@
           └─StreamDml { columns: [x, y, _row_id] }
             └─StreamSource
   with_config_map:
-    enable_shared_source: 'true'
+    streaming_use_shared_source: 'true'
 - before:
   - create_source
   sql: |
-    SET enable_shared_source = false;
+    SET streaming_use_shared_source = false;
     select * from s;
   batch_plan: |-
     BatchExchange { order: [], dist: Single }
@@ -43,11 +43,11 @@
       └─StreamRowIdGen { row_id_index: 3 }
         └─StreamSource { source: s, columns: [x, y, _rw_kafka_timestamp, _row_id] }
   with_config_map:
-    enable_shared_source: 'false'
+    streaming_use_shared_source: 'false'
 - before:
   - create_source
   sql: |
-    SET enable_shared_source = true;
+    SET streaming_use_shared_source = true;
     select * from s;
   batch_plan: |-
     BatchExchange { order: [], dist: Single }
@@ -59,11 +59,11 @@
       └─StreamRowIdGen { row_id_index: 3 }
         └─StreamSource { source: s, columns: [x, y, _rw_kafka_timestamp, _row_id] }
   with_config_map:
-    enable_shared_source: 'false'
+    streaming_use_shared_source: 'false'
 - before:
   - create_source
   sql: |
-    SET enable_shared_source = false;
+    SET streaming_use_shared_source = false;
     select * from s;
   batch_plan: |-
     BatchExchange { order: [], dist: Single }
@@ -75,11 +75,11 @@
       └─StreamRowIdGen { row_id_index: 3 }
         └─StreamSourceScan { columns: [x, y, _rw_kafka_timestamp, _row_id, _rw_kafka_partition, _rw_kafka_offset] }
   with_config_map:
-    enable_shared_source: 'true'
+    streaming_use_shared_source: 'true'
 - before:
   - create_source
   sql: |
-    SET enable_shared_source = true;
+    SET streaming_use_shared_source = true;
     select * from s;
   batch_plan: |-
     BatchExchange { order: [], dist: Single }
@@ -91,4 +91,4 @@
       └─StreamRowIdGen { row_id_index: 3 }
         └─StreamSourceScan { columns: [x, y, _rw_kafka_timestamp, _row_id, _rw_kafka_partition, _rw_kafka_offset] }
   with_config_map:
-    enable_shared_source: 'true'
+    streaming_use_shared_source: 'true'
diff --git a/src/frontend/planner_test/tests/testdata/output/subquery.yaml b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
index 98a0fcbf8fdd3..a33389f889d64 100644
--- a/src/frontend/planner_test/tests/testdata/output/subquery.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/subquery.yaml
@@ -223,11 +223,11 @@
     AND pg_catalog.pg_table_is_visible(c.oid)
     ORDER BY 1,2;
   logical_plan: |-
-    LogicalProject { exprs: [rw_schemas.name, rw_tables.name, Case(($expr1 = 'r':Varchar), 'table':Varchar, ($expr1 = 'v':Varchar), 'view':Varchar, ($expr1 = 'm':Varchar), 'materialized view':Varchar, ($expr1 = 'i':Varchar), 'index':Varchar, ($expr1 = 'S':Varchar), 'sequence':Varchar, ($expr1 = 's':Varchar), 'special':Varchar, ($expr1 = 't':Varchar), 'TOAST table':Varchar, ($expr1 = 'f':Varchar), 'foreign table':Varchar, ($expr1 = 'p':Varchar), 'partitioned table':Varchar, ($expr1 = 'I':Varchar), 'partitioned index':Varchar) as $expr3, PgGetUserbyid(rw_tables.owner) as $expr4] }
+    LogicalProject { exprs: [rw_schemas.name, rw_tables.name, Case(($expr1 = 'r':Varchar), 'table':Varchar, ($expr1 = 'v':Varchar), 'view':Varchar, ($expr1 = 'm':Varchar), 'materialized view':Varchar, ($expr1 = 'i':Varchar), 'index':Varchar, ($expr1 = 'S':Varchar), 'sequence':Varchar, ($expr1 = 's':Varchar), 'special':Varchar, ($expr1 = 't':Varchar), 'TOAST table':Varchar, ($expr1 = 'f':Varchar), 'foreign table':Varchar, ($expr1 = 'p':Varchar), 'partitioned table':Varchar, ($expr1 = 'I':Varchar), 'partitioned index':Varchar) as $expr4, PgGetUserbyid(rw_tables.owner) as $expr5] }
     └─LogicalFilter { predicate: In($expr1, 'r':Varchar, 'p':Varchar, 'v':Varchar, 'm':Varchar, 'S':Varchar, 'f':Varchar, '':Varchar) AND (rw_schemas.name <> 'pg_catalog':Varchar) AND Not(RegexpEq(rw_schemas.name, '^pg_toast':Varchar)) AND (rw_schemas.name <> 'information_schema':Varchar) }
       └─LogicalJoin { type: LeftOuter, on: (rw_schemas.id = rw_tables.schema_id), output: all }
         ├─LogicalShare { id: 18 }
-        │ └─LogicalProject { exprs: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, 'p':Varchar, Case(('table':Varchar = 'table':Varchar), 'r':Varchar, ('table':Varchar = 'system table':Varchar), 'r':Varchar, ('table':Varchar = 'index':Varchar), 'i':Varchar, ('table':Varchar = 'view':Varchar), 'v':Varchar, ('table':Varchar = 'materialized view':Varchar), 'm':Varchar) as $expr1, 0:Int32, 0:Int32, Array as $expr2, false:Boolean, null:Varchar] }
+        │ └─LogicalProject { exprs: [rw_tables.id, rw_tables.name, rw_tables.schema_id, rw_tables.owner, 'p':Varchar, Case(('table':Varchar = 'table':Varchar), 'r':Varchar, ('table':Varchar = 'system table':Varchar), 'r':Varchar, ('table':Varchar = 'index':Varchar), 'i':Varchar, ('table':Varchar = 'view':Varchar), 'v':Varchar, ('table':Varchar = 'materialized view':Varchar), 'm':Varchar) as $expr1, 0:Int32::Int16 as $expr2, 0:Int32, 0:Int32, Array as $expr3, false:Boolean, null:Varchar] }
         │   └─LogicalShare { id: 16 }
         │     └─LogicalUnion { all: true }
         │       ├─LogicalUnion { all: true }
@@ -241,7 +241,7 @@
         │       │ │ │ │ │ │ └─LogicalProject { exprs: [rw_system_tables.id, rw_system_tables.name, 'system table':Varchar, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] }
         │       │ │ │ │ │ │   └─LogicalSysScan { table: rw_system_tables, columns: [rw_system_tables.id, rw_system_tables.name, rw_system_tables.schema_id, rw_system_tables.owner, rw_system_tables.definition, rw_system_tables.acl] }
         │       │ │ │ │ │ └─LogicalProject { exprs: [rw_sources.id, rw_sources.name, 'source':Varchar, rw_sources.schema_id, rw_sources.owner, rw_sources.definition, rw_sources.acl] }
-        │       │ │ │ │ │   └─LogicalSysScan { table: rw_sources, columns: [rw_sources.id, rw_sources.name, rw_sources.schema_id, rw_sources.owner, rw_sources.connector, rw_sources.columns, rw_sources.format, rw_sources.row_encode, rw_sources.append_only, rw_sources.associated_table_id, rw_sources.connection_id, rw_sources.definition, rw_sources.acl, rw_sources.initialized_at, rw_sources.created_at, rw_sources.initialized_at_cluster_version, rw_sources.created_at_cluster_version] }
+        │       │ │ │ │ │   └─LogicalSysScan { table: rw_sources, columns: [rw_sources.id, rw_sources.name, rw_sources.schema_id, rw_sources.owner, rw_sources.connector, rw_sources.columns, rw_sources.format, rw_sources.row_encode, rw_sources.append_only, rw_sources.associated_table_id, rw_sources.connection_id, rw_sources.definition, rw_sources.acl, rw_sources.initialized_at, rw_sources.created_at, rw_sources.initialized_at_cluster_version, rw_sources.created_at_cluster_version, rw_sources.is_shared] }
         │       │ │ │ │ └─LogicalProject { exprs: [rw_indexes.id, rw_indexes.name, 'index':Varchar, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl] }
         │       │ │ │ │   └─LogicalSysScan { table: rw_indexes, columns: [rw_indexes.id, rw_indexes.name, rw_indexes.primary_table_id, rw_indexes.key_columns, rw_indexes.include_columns, rw_indexes.schema_id, rw_indexes.owner, rw_indexes.definition, rw_indexes.acl, rw_indexes.initialized_at, rw_indexes.created_at, rw_indexes.initialized_at_cluster_version, rw_indexes.created_at_cluster_version] }
         │       │ │ │ └─LogicalProject { exprs: [rw_sinks.id, rw_sinks.name, 'sink':Varchar, rw_sinks.schema_id, rw_sinks.owner, rw_sinks.definition, rw_sinks.acl] }
@@ -249,7 +249,7 @@
         │       │ │ └─LogicalProject { exprs: [rw_subscriptions.id, rw_subscriptions.name, 'subscription':Varchar, rw_subscriptions.schema_id, rw_subscriptions.owner, rw_subscriptions.definition, rw_subscriptions.acl] }
         │       │ │   └─LogicalSysScan { table: rw_subscriptions, columns: [rw_subscriptions.id, rw_subscriptions.name, rw_subscriptions.schema_id, rw_subscriptions.owner, rw_subscriptions.definition, rw_subscriptions.acl, rw_subscriptions.initialized_at, rw_subscriptions.created_at, rw_subscriptions.initialized_at_cluster_version, rw_subscriptions.created_at_cluster_version] }
         │       │ └─LogicalProject { exprs: [rw_materialized_views.id, rw_materialized_views.name, 'materialized view':Varchar, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.acl] }
-        │       │   └─LogicalSysScan { table: rw_materialized_views, columns: [rw_materialized_views.id, rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.append_only, rw_materialized_views.acl, rw_materialized_views.initialized_at, rw_materialized_views.created_at, rw_materialized_views.initialized_at_cluster_version, rw_materialized_views.created_at_cluster_version] }
+        │       │   └─LogicalSysScan { table: rw_materialized_views, columns: [rw_materialized_views.id, rw_materialized_views.name, rw_materialized_views.schema_id, rw_materialized_views.owner, rw_materialized_views.definition, rw_materialized_views.append_only, rw_materialized_views.acl, rw_materialized_views.initialized_at, rw_materialized_views.created_at, rw_materialized_views.initialized_at_cluster_version, rw_materialized_views.created_at_cluster_version, rw_materialized_views.background_ddl] }
         │       └─LogicalProject { exprs: [rw_views.id, rw_views.name, 'view':Varchar, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] }
         │         └─LogicalSysScan { table: rw_views, columns: [rw_views.id, rw_views.name, rw_views.schema_id, rw_views.owner, rw_views.definition, rw_views.acl] }
         └─LogicalShare { id: 20 }
diff --git a/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml b/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml
index 09905835f2009..13d363a6e4877 100644
--- a/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/subquery_expr_correlated.yaml
@@ -2122,3 +2122,100 @@
                   │       └─StreamTableScan { table: t1, columns: [t1.a, t1.b, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
                   └─StreamExchange { dist: HashShard(t6.l) }
                     └─StreamTableScan { table: t6, columns: [t6.k, t6.l, t6._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t6._row_id], pk: [_row_id], dist: UpstreamHashShard(t6._row_id) }
+- name: improve join transpose rule to handle join with output indices
+  sql: |
+    with rawdata as (
+      select 'first' as w, '{"x":{"value":123},"y":{"value":[1,2,3]},"z":{"value":[{"a":4,"b":5},{"a":6,"b":7}]}}'::jsonb as rawjson
+      union all
+      select 'second', '{"x":{"value":456},"y":{"value":[7,8,9]},"z":{"value":[{"a":0,"b":1},{"a":2,"b":3}]}}'::jsonb
+    )
+    select
+      array(
+        select
+          array(
+            select 1
+            from jsonb_array_elements(
+              case
+                when jsonb_typeof(o.value->'value') = 'array' then o.value->'value'
+                else '[]'::jsonb
+              end
+            ) with ordinality as x(v, i), jsonb_each (
+              case
+                when jsonb_typeof(v) = 'object' then v
+                else '{}'::jsonb
+              end
+            ) as j
+        )
+        from jsonb_each(rawjson) AS o
+        where jsonb_typeof(o.value) = 'object' and o.value ? 'value'
+      ) as kv
+    from rawdata
+  optimized_logical_plan_for_batch: |-
+    LogicalJoin { type: LeftOuter, on: IsNotDistinctFrom('{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, '{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb), output: [$expr6] }
+    ├─LogicalValues { rows: [['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], ['{"x": {"value": 456}, "y": {"value": [7, 8, 9]}, "z": {"value": [{"a": 0, "b": 1}, {"a": 2, "b": 3}]}}':Jsonb]], schema: Schema { fields: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb:Jsonb] } }
+    └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, Coalesce(array_agg($expr5) filter(IsNotNull(1:Int32)), ARRAY[]:List(List(Int32))) as $expr6] }
+      └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], aggs: [array_agg($expr5) filter(IsNotNull(1:Int32))] }
+        └─LogicalJoin { type: LeftOuter, on: IsNotDistinctFrom('{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, '{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb), output: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr5, 1:Int32] }
+          ├─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], aggs: [] }
+          │ └─LogicalValues { rows: [['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], ['{"x": {"value": 456}, "y": {"value": [7, 8, 9]}, "z": {"value": [{"a": 0, "b": 1}, {"a": 2, "b": 3}]}}':Jsonb]], schema: Schema { fields: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb:Jsonb] } }
+          └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr5, 1:Int32] }
+            └─LogicalJoin { type: LeftOuter, on: IsNotDistinctFrom($expr1, $expr2) AND IsNotDistinctFrom('{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, '{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb), output: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr5] }
+              ├─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, Field(JsonbEach($0), 1:Int32) as $expr1] }
+              │ └─LogicalFilter { predicate: (JsonbTypeof(Field(JsonbEach($0), 1:Int32)) = 'object':Varchar) AND JsonbExists(Field(JsonbEach($0), 1:Int32), 'value':Varchar) }
+              │   └─LogicalProjectSet { select_list: [$0, JsonbEach($0)] }
+              │     └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], aggs: [] }
+              │       └─LogicalValues { rows: [['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], ['{"x": {"value": 456}, "y": {"value": [7, 8, 9]}, "z": {"value": [{"a": 0, "b": 1}, {"a": 2, "b": 3}]}}':Jsonb]], schema: Schema { fields: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb:Jsonb] } }
+              └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr2, Coalesce(array_agg(1:Int32) filter(IsNotNull(1:Int32)), ARRAY[]:List(Int32)) as $expr5] }
+                └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr2], aggs: [array_agg(1:Int32) filter(IsNotNull(1:Int32))] }
+                  └─LogicalJoin { type: LeftOuter, on: IsNotDistinctFrom($expr2, $expr3) AND IsNotDistinctFrom('{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, '{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb), output: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr2, 1:Int32, 1:Int32] }
+                    ├─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr2], aggs: [] }
+                    │ └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, Field(JsonbEach($0), 1:Int32) as $expr2] }
+                    │   └─LogicalFilter { predicate: (JsonbTypeof(Field(JsonbEach($0), 1:Int32)) = 'object':Varchar) AND JsonbExists(Field(JsonbEach($0), 1:Int32), 'value':Varchar) }
+                    │     └─LogicalProjectSet { select_list: [$0, JsonbEach($0)] }
+                    │       └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], aggs: [] }
+                    │         └─LogicalValues { rows: [['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], ['{"x": {"value": 456}, "y": {"value": [7, 8, 9]}, "z": {"value": [{"a": 0, "b": 1}, {"a": 2, "b": 3}]}}':Jsonb]], schema: Schema { fields: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb:Jsonb] } }
+                    └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr3, 1:Int32, 1:Int32] }
+                      └─LogicalJoin { type: Inner, on: IsNotDistinctFrom(JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb)), JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb))) AND IsNotDistinctFrom(JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb)), JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb))) AND IsNotDistinctFrom($expr3, $expr4) AND IsNotDistinctFrom('{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, '{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb), output: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr3] }
+                        ├─LogicalProjectSet { select_list: [$0, $1, JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb))] }
+                        │ └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr3], aggs: [] }
+                        │   └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, Field(JsonbEach($0), 1:Int32) as $expr3] }
+                        │     └─LogicalFilter { predicate: (JsonbTypeof(Field(JsonbEach($0), 1:Int32)) = 'object':Varchar) AND JsonbExists(Field(JsonbEach($0), 1:Int32), 'value':Varchar) }
+                        │       └─LogicalProjectSet { select_list: [$0, JsonbEach($0)] }
+                        │         └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], aggs: [] }
+                        │           └─LogicalValues { rows: [['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], ['{"x": {"value": 456}, "y": {"value": [7, 8, 9]}, "z": {"value": [{"a": 0, "b": 1}, {"a": 2, "b": 3}]}}':Jsonb]], schema: Schema { fields: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb:Jsonb] } }
+                        └─LogicalProjectSet { select_list: [$0, $1, $2, $3, JsonbEach(Case((JsonbTypeof($3) = 'object':Varchar), $3, '{}':Jsonb))] }
+                          └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr4, JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb)), JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb))] }
+                            └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr4, JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb))], aggs: [] }
+                              └─LogicalProjectSet { select_list: [$0, $1, JsonbArrayElements(Case((JsonbTypeof(JsonbAccess($1, 'value':Varchar)) = 'array':Varchar), JsonbAccess($1, 'value':Varchar), '[]':Jsonb))] }
+                                └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, $expr4], aggs: [] }
+                                  └─LogicalProject { exprs: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb, Field(JsonbEach($0), 1:Int32) as $expr4] }
+                                    └─LogicalFilter { predicate: (JsonbTypeof(Field(JsonbEach($0), 1:Int32)) = 'object':Varchar) AND JsonbExists(Field(JsonbEach($0), 1:Int32), 'value':Varchar) }
+                                      └─LogicalProjectSet { select_list: [$0, JsonbEach($0)] }
+                                        └─LogicalAgg { group_key: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], aggs: [] }
+                                          └─LogicalValues { rows: [['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb], ['{"x": {"value": 456}, "y": {"value": [7, 8, 9]}, "z": {"value": [{"a": 0, "b": 1}, {"a": 2, "b": 3}]}}':Jsonb]], schema: Schema { fields: ['{"x": {"value": 123}, "y": {"value": [1, 2, 3]}, "z": {"value": [{"a": 4, "b": 5}, {"a": 6, "b": 7}]}}':Jsonb:Jsonb] } }
+- name: subquery with CTE
+  sql: |
+    create table t1(x int, y int);
+    create table t2(x int, y int);
+    select * from t1 where t1.x = ( with cte as (select * from t2 where t2.y = t1.y)  select x from cte limit 1);
+  batch_plan: |-
+    BatchExchange { order: [], dist: Single }
+    └─BatchHashJoin { type: Inner, predicate: t1.y IS NOT DISTINCT FROM t2.y AND t1.x = t2.x, output: [t1.x, t1.y] }
+      ├─BatchExchange { order: [], dist: HashShard(t1.y) }
+      │ └─BatchScan { table: t1, columns: [t1.x, t1.y], distribution: SomeShard }
+      └─BatchGroupTopN { order: [t2.y ASC], limit: 1, offset: 0, group_key: [t2.y] }
+        └─BatchExchange { order: [], dist: HashShard(t2.y) }
+          └─BatchProject { exprs: [t2.y, t2.x] }
+            └─BatchFilter { predicate: IsNotNull(t2.y) }
+              └─BatchScan { table: t2, columns: [t2.x, t2.y], distribution: SomeShard }
+  stream_plan: |-
+    StreamMaterialize { columns: [x, y, t1._row_id(hidden), t2.y(hidden)], stream_key: [t1._row_id, y, x], pk_columns: [t1._row_id, y, x], pk_conflict: NoCheck }
+    └─StreamExchange { dist: HashShard(t1.x, t1.y, t1._row_id) }
+      └─StreamHashJoin { type: Inner, predicate: t1.y IS NOT DISTINCT FROM t2.y AND t1.x = t2.x, output: [t1.x, t1.y, t1._row_id, t2.y] }
+        ├─StreamExchange { dist: HashShard(t1.y) }
+        │ └─StreamTableScan { table: t1, columns: [t1.x, t1.y, t1._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t1._row_id], pk: [_row_id], dist: UpstreamHashShard(t1._row_id) }
+        └─StreamGroupTopN { order: [t2.y ASC], limit: 1, offset: 0, group_key: [t2.y] }
+          └─StreamExchange { dist: HashShard(t2.y) }
+            └─StreamProject { exprs: [t2.y, t2.x, t2._row_id] }
+              └─StreamFilter { predicate: IsNotNull(t2.y) }
+                └─StreamTableScan { table: t2, columns: [t2.x, t2.y, t2._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t2._row_id], pk: [_row_id], dist: UpstreamHashShard(t2._row_id) }
diff --git a/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml b/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml
index 514a56f7dff6d..4ad48902ffb05 100644
--- a/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/temporal_filter.yaml
@@ -331,7 +331,7 @@
     StreamMaterialize { columns: [id1, a1, id2, v1, stream._row_id(hidden)], stream_key: [stream._row_id, id1], pk_columns: [stream._row_id, id1], pk_conflict: NoCheck, watermark_columns: [v1] }
     └─StreamProject { exprs: [stream.id1, stream.a1, version.id2, stream.v1, stream._row_id], output_watermarks: [stream.v1] }
       └─StreamDynamicFilter { predicate: (stream.v1 > now), output_watermarks: [stream.v1], output: [stream.id1, stream.a1, stream.v1, version.id2, stream._row_id], cleaned_by_watermark: true }
-        ├─StreamTemporalJoin { type: LeftOuter, append_only: true, predicate: stream.id1 = version.id2, output: [stream.id1, stream.a1, stream.v1, version.id2, stream._row_id] }
+        ├─StreamTemporalJoin { type: LeftOuter, append_only: true, predicate: stream.id1 = version.id2, nested_loop: false, output: [stream.id1, stream.a1, stream.v1, version.id2, stream._row_id] }
         │ ├─StreamExchange { dist: HashShard(stream.id1) }
         │ │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream.v1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         │ └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
diff --git a/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml b/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml
index 40fda8fcb85ab..41aad6991429a 100644
--- a/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml
+++ b/src/frontend/planner_test/tests/testdata/output/temporal_join.yaml
@@ -7,7 +7,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden)], stream_key: [stream._row_id, id1], pk_columns: [stream._row_id, id1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream._row_id) }
-      └─StreamTemporalJoin { type: LeftOuter, append_only: true, predicate: stream.id1 = version.id2, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
+      └─StreamTemporalJoin { type: LeftOuter, append_only: true, predicate: stream.id1 = version.id2, nested_loop: false, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
         ├─StreamExchange { dist: HashShard(stream.id1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
@@ -23,7 +23,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden)], stream_key: [stream._row_id, id1], pk_columns: [stream._row_id, id1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), nested_loop: false, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
         ├─StreamExchange { dist: HashShard(stream.id1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
@@ -36,7 +36,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden)], stream_key: [stream._row_id, id1], pk_columns: [stream._row_id, id1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), nested_loop: false, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
         ├─StreamExchange { dist: HashShard(stream.id1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
@@ -49,7 +49,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden)], stream_key: [stream._row_id, id1, a1], pk_columns: [stream._row_id, id1, a1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream.a1, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND stream.a1 = version.a2 AND (version.b2 <> version.a2), output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND stream.a1 = version.a2 AND (version.b2 <> version.a2), nested_loop: false, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
         ├─StreamExchange { dist: HashShard(stream.id1, stream.a1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2, version.a2) }
@@ -65,7 +65,7 @@
       └─StreamSimpleAgg [append_only] { aggs: [sum0(count), count] }
         └─StreamExchange { dist: Single }
           └─StreamStatelessSimpleAgg { aggs: [count] }
-            └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), output: [stream._row_id, stream.id1, version.id2] }
+            └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), nested_loop: false, output: [stream._row_id, stream.id1, version.id2] }
               ├─StreamExchange { dist: HashShard(stream.id1) }
               │ └─StreamTableScan { table: stream, columns: [stream.id1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
               └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
@@ -86,7 +86,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden)], stream_key: [stream._row_id, id1], pk_columns: [stream._row_id, id1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
+      └─StreamTemporalJoin { type: Inner, append_only: true, predicate: stream.id1 = version.id2 AND (version.a2 < 10:Int32), nested_loop: false, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id] }
         ├─StreamExchange { dist: HashShard(stream.id1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
@@ -111,9 +111,9 @@
   stream_plan: |-
     StreamMaterialize { columns: [k, x1, x2, a1, b1, stream._row_id(hidden), version2.k(hidden)], stream_key: [stream._row_id, k], pk_columns: [stream._row_id, k], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.k, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.k = version2.k, output: [stream.k, version1.x1, version2.x2, stream.a1, stream.b1, stream._row_id, version2.k] }
+      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.k = version2.k, nested_loop: false, output: [stream.k, version1.x1, version2.x2, stream.a1, stream.b1, stream._row_id, version2.k] }
         ├─StreamExchange { dist: HashShard(stream.k) }
-        │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.k = version1.k, output: [stream.k, stream.a1, stream.b1, version1.x1, stream._row_id, version1.k] }
+        │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.k = version1.k, nested_loop: false, output: [stream.k, stream.a1, stream.b1, version1.x1, stream._row_id, version1.k] }
         │   ├─StreamExchange { dist: HashShard(stream.k) }
         │   │ └─StreamFilter { predicate: (stream.a1 < 10:Int32) }
         │   │   └─StreamTableScan { table: stream, columns: [stream.k, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
@@ -133,9 +133,9 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, x1, id2, x2, a1, b1, stream._row_id(hidden), version2.id2(hidden)], stream_key: [stream._row_id, id1, id2], pk_columns: [stream._row_id, id1, id2], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream.id2, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id2 = version2.id2, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
+      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id2 = version2.id2, nested_loop: false, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
         ├─StreamExchange { dist: HashShard(stream.id2) }
-        │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id1 = version1.id1, output: [stream.id1, stream.id2, stream.a1, stream.b1, version1.x1, stream._row_id, version1.id1] }
+        │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id1 = version1.id1, nested_loop: false, output: [stream.id1, stream.id2, stream.a1, stream.b1, version1.x1, stream._row_id, version1.id1] }
         │   ├─StreamExchange { dist: HashShard(stream.id1) }
         │   │ └─StreamFilter { predicate: (stream.a1 < 10:Int32) }
         │   │   └─StreamTableScan { table: stream, columns: [stream.id1, stream.id2, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
@@ -155,9 +155,9 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, x1, id2, x2, a1, b1, stream._row_id(hidden), version2.id2(hidden)], stream_key: [stream._row_id, id1, id2], pk_columns: [stream._row_id, id1, id2], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream.id2, stream._row_id) }
-      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id2 = version2.id2, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
+      └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id2 = version2.id2, nested_loop: false, output: [stream.id1, version1.x1, stream.id2, version2.x2, stream.a1, stream.b1, stream._row_id, version2.id2] }
         ├─StreamExchange { dist: HashShard(stream.id2) }
-        │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id1 = version1.id1, output: [stream.id1, stream.id2, stream.a1, stream.b1, version1.x1, stream._row_id, version1.id1] }
+        │ └─StreamTemporalJoin { type: Inner, append_only: false, predicate: stream.id1 = version1.id1, nested_loop: false, output: [stream.id1, stream.id2, stream.a1, stream.b1, version1.x1, stream._row_id, version1.id1] }
         │   ├─StreamExchange { dist: HashShard(stream.id1) }
         │   │ └─StreamFilter { predicate: (stream.a1 < 10:Int32) }
         │   │   └─StreamTableScan { table: stream, columns: [stream.id1, stream.id2, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
@@ -174,7 +174,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), stream.b1(hidden)], stream_key: [stream._row_id, id2, a1, stream.b1], pk_columns: [stream._row_id, id2, a1, stream.b1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.a1, idx2.id2, stream._row_id, stream.b1) }
-      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx2.a2 AND stream.b1 = idx2.b2, output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
+      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx2.a2 AND stream.b1 = idx2.b2, nested_loop: false, output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
         ├─StreamExchange { dist: HashShard(stream.a1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(idx2.a2) }
@@ -188,7 +188,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), stream.b1(hidden)], stream_key: [stream._row_id, id2, a1, stream.b1], pk_columns: [stream._row_id, id2, a1, stream.b1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.a1, idx2.id2, stream._row_id, stream.b1) }
-      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx2.a2 AND stream.b1 = idx2.b2, output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
+      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx2.a2 AND stream.b1 = idx2.b2, nested_loop: false, output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
         ├─StreamExchange { dist: HashShard(stream.a1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(idx2.a2) }
@@ -202,7 +202,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), stream.b1(hidden)], stream_key: [stream._row_id, id2, stream.b1, a1], pk_columns: [stream._row_id, id2, stream.b1, a1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.a1, idx2.id2, stream._row_id, stream.b1) }
-      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.b1 = idx2.b2 AND (stream.a1 = idx2.a2), output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
+      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.b1 = idx2.b2 AND (stream.a1 = idx2.a2), nested_loop: false, output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
         ├─StreamExchange { dist: HashShard(stream.b1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(idx2.b2) }
@@ -214,7 +214,7 @@
     select * from t left join v FOR SYSTEM_TIME AS OF PROCTIME() on a = count;
   stream_plan: |-
     StreamMaterialize { columns: [a, count, t._row_id(hidden), $expr1(hidden)], stream_key: [t._row_id, $expr1], pk_columns: [t._row_id, $expr1], pk_conflict: NoCheck }
-    └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate:  AND ($expr1 = v.count), output: [t.a, v.count, t._row_id, $expr1] }
+    └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate:  AND ($expr1 = v.count), nested_loop: false, output: [t.a, v.count, t._row_id, $expr1] }
       ├─StreamExchange { dist: Single }
       │ └─StreamProject { exprs: [t.a, t.a::Int64 as $expr1, t._row_id] }
       │   └─StreamTableScan { table: t, columns: [t.a, t._row_id], stream_scan_type: ArrangementBackfill, stream_key: [t._row_id], pk: [_row_id], dist: UpstreamHashShard(t._row_id) }
@@ -229,7 +229,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), stream.b1(hidden)], stream_key: [stream._row_id, id2, a1, stream.b1], pk_columns: [stream._row_id, id2, a1, stream.b1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.a1, idx.id2, stream._row_id, stream.b1) }
-      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx.a2 AND stream.b1 = idx.b2, output: [stream.id1, stream.a1, idx.id2, idx.a2, stream._row_id, stream.b1] }
+      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx.a2 AND stream.b1 = idx.b2, nested_loop: false, output: [stream.id1, stream.a1, idx.id2, idx.a2, stream._row_id, stream.b1] }
         ├─StreamExchange { dist: HashShard(stream.a1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(idx.a2) }
@@ -244,7 +244,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), stream.b1(hidden)], stream_key: [stream._row_id, id2, a1, stream.b1], pk_columns: [stream._row_id, id2, a1, stream.b1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.a1, idx2.id2, stream._row_id, stream.b1) }
-      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx2.a2 AND stream.b1 = idx2.b2, output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
+      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.a1 = idx2.a2 AND stream.b1 = idx2.b2, nested_loop: false, output: [stream.id1, stream.a1, idx2.id2, idx2.a2, stream._row_id, stream.b1] }
         ├─StreamExchange { dist: HashShard(stream.a1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream.b1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(idx2.a2) }
@@ -260,7 +260,7 @@
   stream_plan: |-
     StreamMaterialize { columns: [id1, a1, id2, a2, stream._row_id(hidden), stream.b1(hidden), stream.c1(hidden)], stream_key: [stream._row_id, id1, a1, stream.b1, stream.c1], pk_columns: [stream._row_id, id1, a1, stream.b1, stream.c1], pk_conflict: NoCheck }
     └─StreamExchange { dist: HashShard(stream.id1, stream.a1, stream._row_id, stream.b1, stream.c1) }
-      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.id1 = version.id2 AND (stream.a1 = version.a2) AND (stream.b1 = version.b2) AND (stream.c1 = version.c2), output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id, stream.b1, stream.c1] }
+      └─StreamTemporalJoin { type: LeftOuter, append_only: false, predicate: stream.id1 = version.id2 AND (stream.a1 = version.a2) AND (stream.b1 = version.b2) AND (stream.c1 = version.c2), nested_loop: false, output: [stream.id1, stream.a1, version.id2, version.a2, stream._row_id, stream.b1, stream.c1] }
         ├─StreamExchange { dist: HashShard(stream.id1) }
         │ └─StreamTableScan { table: stream, columns: [stream.id1, stream.a1, stream.b1, stream.c1, stream._row_id], stream_scan_type: ArrangementBackfill, stream_key: [stream._row_id], pk: [_row_id], dist: UpstreamHashShard(stream._row_id) }
         └─StreamExchange [no_shuffle] { dist: UpstreamHashShard(version.id2) }
diff --git a/src/frontend/src/binder/expr/column.rs b/src/frontend/src/binder/expr/column.rs
index cf4f295e960a0..8a8ea6b87f7f5 100644
--- a/src/frontend/src/binder/expr/column.rs
+++ b/src/frontend/src/binder/expr/column.rs
@@ -151,14 +151,14 @@ impl Binder {
                 }
             }
 
-            for (i, lateral_context) in lateral_contexts.iter().rev().enumerate() {
+            for (j, lateral_context) in lateral_contexts.iter().rev().enumerate() {
                 if lateral_context.is_visible {
                     let context = &lateral_context.context;
                     if matches!(context.clause, Some(Clause::Insert)) {
                         continue;
                     }
                     // correlated input ref from lateral context `depth` starts from 1.
-                    let depth = i + 1;
+                    let depth = i + j + 1;
                     match context.get_column_binding_index(&table_name, &column_name) {
                         Ok(index) => {
                             let column = &context.columns[index];
diff --git a/src/frontend/src/binder/expr/function/builtin_scalar.rs b/src/frontend/src/binder/expr/function/builtin_scalar.rs
index 7781140432577..68b37a3fee4e0 100644
--- a/src/frontend/src/binder/expr/function/builtin_scalar.rs
+++ b/src/frontend/src/binder/expr/function/builtin_scalar.rs
@@ -432,10 +432,10 @@ impl Binder {
                 ("current_catalog", current_database()),
                 ("current_database", current_database()),
                 ("current_schema", guard_by_len(0, raw(|binder, _inputs| {
-                    return Ok(binder
+                    Ok(binder
                         .first_valid_schema()
                         .map(|schema| ExprImpl::literal_varchar(schema.name()))
-                        .unwrap_or_else(|_| ExprImpl::literal_null(DataType::Varchar)));
+                        .unwrap_or_else(|_| ExprImpl::literal_null(DataType::Varchar)))
                 }))),
                 ("current_schemas", raw(|binder, mut inputs| {
                     let no_match_err = ErrorCode::ExprError(
diff --git a/src/frontend/src/binder/expr/function/mod.rs b/src/frontend/src/binder/expr/function/mod.rs
index ddc21c6ee7ac7..f7a4007ffd467 100644
--- a/src/frontend/src/binder/expr/function/mod.rs
+++ b/src/frontend/src/binder/expr/function/mod.rs
@@ -333,6 +333,17 @@ impl Binder {
                     .context("postgres_query error")?
                     .into());
             }
+            // `mysql_query` table function
+            if func_name.eq("mysql_query") {
+                reject_syntax!(
+                    arg_list.variadic,
+                    "`VARIADIC` is not allowed in table function call"
+                );
+                self.ensure_table_function_allowed()?;
+                return Ok(TableFunction::new_mysql_query(args)
+                    .context("mysql_query error")?
+                    .into());
+            }
             // UDTF
             if let Some(ref udf) = udf
                 && udf.kind.is_table()
diff --git a/src/frontend/src/binder/insert.rs b/src/frontend/src/binder/insert.rs
index 505acec2265b7..ac4a52f2c11a4 100644
--- a/src/frontend/src/binder/insert.rs
+++ b/src/frontend/src/binder/insert.rs
@@ -305,7 +305,6 @@ impl Binder {
 
     /// Cast a list of `exprs` to corresponding `expected_types` IN ASSIGNMENT CONTEXT. Make sure
     /// you understand the difference of implicit, assignment and explicit cast before reusing it.
-
     pub(super) fn cast_on_insert(
         expected_types: &Vec<DataType>,
         exprs: Vec<ExprImpl>,
diff --git a/src/frontend/src/binder/query.rs b/src/frontend/src/binder/query.rs
index 7ad2091e6fb87..a6d39315c51c1 100644
--- a/src/frontend/src/binder/query.rs
+++ b/src/frontend/src/binder/query.rs
@@ -20,7 +20,7 @@ use risingwave_common::catalog::Schema;
 use risingwave_common::types::DataType;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_sqlparser::ast::{
-    Cte, CteInner, Expr, Fetch, ObjectName, OrderByExpr, Query, SetExpr, SetOperator, Value, With,
+    Cte, CteInner, Expr, Fetch, OrderByExpr, Query, SetExpr, SetOperator, Value, With,
 };
 use thiserror_ext::AsReport;
 
@@ -350,11 +350,8 @@ impl Binder {
                     }
                     CteInner::ChangeLog(from_table_name) => {
                         self.push_context();
-                        let from_table_relation = self.bind_relation_by_name(
-                            ObjectName::from(vec![from_table_name]),
-                            None,
-                            None,
-                        )?;
+                        let from_table_relation =
+                            self.bind_relation_by_name(from_table_name.clone(), None, None)?;
                         self.pop_context()?;
                         self.context.cte_to_relation.insert(
                             table_name,
diff --git a/src/frontend/src/binder/relation/mod.rs b/src/frontend/src/binder/relation/mod.rs
index af4121e1b9056..861edf575f510 100644
--- a/src/frontend/src/binder/relation/mod.rs
+++ b/src/frontend/src/binder/relation/mod.rs
@@ -15,6 +15,7 @@
 use std::collections::hash_map::Entry;
 use std::ops::Deref;
 
+use either::Either;
 use itertools::{EitherOrBoth, Itertools};
 use risingwave_common::bail;
 use risingwave_common::catalog::{Field, TableId, DEFAULT_SCHEMA_NAME};
@@ -134,6 +135,15 @@ impl Relation {
                 with_ordinality: _,
             } => table_function
                 .collect_correlated_indices_by_depth_and_assign_id(depth + 1, correlated_id),
+            Relation::Share(share) => match &mut share.input {
+                BoundShareInput::Query(query) => match query {
+                    Either::Left(query) => query
+                        .collect_correlated_indices_by_depth_and_assign_id(depth, correlated_id),
+                    Either::Right(_) => vec![],
+                },
+                BoundShareInput::ChangeLog(change_log) => change_log
+                    .collect_correlated_indices_by_depth_and_assign_id(depth, correlated_id),
+            },
             _ => vec![],
         }
     }
diff --git a/src/frontend/src/binder/set_expr.rs b/src/frontend/src/binder/set_expr.rs
index 68af5845bf7a4..0d7f74efa95d9 100644
--- a/src/frontend/src/binder/set_expr.rs
+++ b/src/frontend/src/binder/set_expr.rs
@@ -78,7 +78,6 @@ impl From<SetOperator> for BoundSetOperation {
 
 impl BoundSetExpr {
     /// The schema returned by this [`BoundSetExpr`].
-
     pub fn schema(&self) -> Cow<'_, Schema> {
         match self {
             BoundSetExpr::Select(s) => Cow::Borrowed(s.schema()),
diff --git a/src/frontend/src/catalog/catalog_service.rs b/src/frontend/src/catalog/catalog_service.rs
index 2d3fbd7e0178a..271d395181df8 100644
--- a/src/frontend/src/catalog/catalog_service.rs
+++ b/src/frontend/src/catalog/catalog_service.rs
@@ -25,8 +25,9 @@ use risingwave_pb::catalog::{
     PbSubscription, PbTable, PbView,
 };
 use risingwave_pb::ddl_service::{
-    alter_name_request, alter_owner_request, alter_set_schema_request, create_connection_request,
-    PbReplaceTablePlan, PbTableJobType, ReplaceTablePlan, TableJobType, WaitVersion,
+    alter_name_request, alter_owner_request, alter_set_schema_request, alter_swap_rename_request,
+    create_connection_request, PbReplaceTablePlan, PbTableJobType, ReplaceTablePlan, TableJobType,
+    WaitVersion,
 };
 use risingwave_pb::meta::PbTableParallelism;
 use risingwave_pb::stream_plan::StreamFragmentGraph;
@@ -197,6 +198,8 @@ pub trait CatalogWriter: Send + Sync {
         object: alter_set_schema_request::Object,
         new_schema_id: u32,
     ) -> Result<()>;
+
+    async fn alter_swap_rename(&self, object: alter_swap_rename_request::Object) -> Result<()>;
 }
 
 #[derive(Clone)]
@@ -498,6 +501,11 @@ impl CatalogWriter for CatalogWriterImpl {
 
         Ok(())
     }
+
+    async fn alter_swap_rename(&self, object: alter_swap_rename_request::Object) -> Result<()> {
+        let version = self.meta_client.alter_swap_rename(object).await?;
+        self.wait_version(version).await
+    }
 }
 
 impl CatalogWriterImpl {
diff --git a/src/frontend/src/catalog/connection_catalog.rs b/src/frontend/src/catalog/connection_catalog.rs
index 54e1210979fe8..03b2ff4203c53 100644
--- a/src/frontend/src/catalog/connection_catalog.rs
+++ b/src/frontend/src/catalog/connection_catalog.rs
@@ -12,18 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
-use std::sync::Arc;
-
-use anyhow::anyhow;
-use risingwave_connector::source::kafka::private_link::insert_privatelink_broker_rewrite_map;
-use risingwave_connector::WithPropertiesExt;
-use risingwave_pb::catalog::connection::private_link_service::PrivateLinkProvider;
 use risingwave_pb::catalog::connection::Info;
 use risingwave_pb::catalog::{connection, PbConnection};
 
 use crate::catalog::{ConnectionId, OwnedByUserCatalog};
-use crate::error::{Result, RwError};
 use crate::user::UserId;
 
 #[derive(Clone, Debug, PartialEq)]
@@ -64,24 +56,3 @@ impl OwnedByUserCatalog for ConnectionCatalog {
         self.owner
     }
 }
-
-pub(crate) fn resolve_private_link_connection(
-    connection: &Arc<ConnectionCatalog>,
-    properties: &mut BTreeMap<String, String>,
-) -> Result<()> {
-    #[allow(irrefutable_let_patterns)]
-    if let connection::Info::PrivateLinkService(svc) = &connection.info {
-        if !properties.is_kafka_connector() {
-            return Err(RwError::from(anyhow!(
-                "Private link is only supported for Kafka connector"
-            )));
-        }
-        // skip all checks for mock connection
-        if svc.get_provider()? == PrivateLinkProvider::Mock {
-            return Ok(());
-        }
-        insert_privatelink_broker_rewrite_map(properties, Some(svc), None)
-            .map_err(RwError::from)?;
-    }
-    Ok(())
-}
diff --git a/src/frontend/src/catalog/schema_catalog.rs b/src/frontend/src/catalog/schema_catalog.rs
index 0394da2a70f81..86d0353f29073 100644
--- a/src/frontend/src/catalog/schema_catalog.rs
+++ b/src/frontend/src/catalog/schema_catalog.rs
@@ -115,10 +115,14 @@ impl SchemaCatalog {
         let table_ref = Arc::new(table);
 
         let old_table = self.table_by_id.get(&id).unwrap();
-        // check if table name get updated.
-        if old_table.name() != name {
+        // check if the table name gets updated.
+        if old_table.name() != name
+            && let Some(t) = self.table_by_name.get(old_table.name())
+            && t.id == id
+        {
             self.table_by_name.remove(old_table.name());
         }
+
         self.table_by_name.insert(name, table_ref.clone());
         self.table_by_id.insert(id, table_ref.clone());
         table_ref
@@ -137,8 +141,11 @@ impl SchemaCatalog {
         let index: IndexCatalog = IndexCatalog::build_from(prost, index_table, primary_table);
         let index_ref = Arc::new(index);
 
-        // check if index name get updated.
-        if old_index.name != name {
+        // check if the index name gets updated.
+        if old_index.name != name
+            && let Some(idx) = self.index_by_name.get(&old_index.name)
+            && idx.id == id
+        {
             self.index_by_name.remove(&old_index.name);
         }
         self.index_by_name.insert(name, index_ref.clone());
@@ -245,8 +252,11 @@ impl SchemaCatalog {
         let source_ref = Arc::new(source);
 
         let old_source = self.source_by_id.get(&id).unwrap();
-        // check if source name get updated.
-        if old_source.name != name {
+        // check if the source name gets updated.
+        if old_source.name != name
+            && let Some(src) = self.source_by_name.get(&old_source.name)
+            && src.id == id
+        {
             self.source_by_name.remove(&old_source.name);
         }
 
@@ -294,8 +304,11 @@ impl SchemaCatalog {
         let sink_ref = Arc::new(sink);
 
         let old_sink = self.sink_by_id.get(&id).unwrap();
-        // check if sink name get updated.
-        if old_sink.name != name {
+        // check if the sink name gets updated.
+        if old_sink.name != name
+            && let Some(s) = self.sink_by_name.get(&old_sink.name)
+            && s.id.sink_id == id
+        {
             self.sink_by_name.remove(&old_sink.name);
         }
 
@@ -331,8 +344,11 @@ impl SchemaCatalog {
         let subscription_ref = Arc::new(subscription);
 
         let old_subscription = self.subscription_by_id.get(&id).unwrap();
-        // check if subscription name get updated.
-        if old_subscription.name != name {
+        // check if the subscription name gets updated.
+        if old_subscription.name != name
+            && let Some(s) = self.subscription_by_name.get(&old_subscription.name)
+            && s.id.subscription_id == id
+        {
             self.subscription_by_name.remove(&old_subscription.name);
         }
 
@@ -365,8 +381,11 @@ impl SchemaCatalog {
         let view_ref = Arc::new(view);
 
         let old_view = self.view_by_id.get(&id).unwrap();
-        // check if view name get updated.
-        if old_view.name != name {
+        // check if the view name gets updated.
+        if old_view.name != name
+            && let Some(v) = self.view_by_name.get(old_view.name())
+            && v.id == id
+        {
             self.view_by_name.remove(&old_view.name);
         }
 
@@ -438,8 +457,11 @@ impl SchemaCatalog {
             .function_by_name
             .get_mut(&old_function_by_id.name)
             .unwrap();
-        // check if function name get updated.
-        if old_function_by_id.name != name {
+        // check if the function name gets updated.
+        if old_function_by_id.name != name
+            && let Some(f) = old_function_by_name.get(&old_function_by_id.arg_types)
+            && f.id == id
+        {
             old_function_by_name.remove(&old_function_by_id.arg_types);
             if old_function_by_name.is_empty() {
                 self.function_by_name.remove(&old_function_by_id.name);
@@ -473,8 +495,11 @@ impl SchemaCatalog {
         let connection_ref = Arc::new(connection);
 
         let old_connection = self.connection_by_id.get(&id).unwrap();
-        // check if connection name get updated.
-        if old_connection.name != name {
+        // check if the connection name gets updated.
+        if old_connection.name != name
+            && let Some(conn) = self.connection_by_name.get(&old_connection.name)
+            && conn.id == id
+        {
             self.connection_by_name.remove(&old_connection.name);
         }
 
@@ -513,8 +538,11 @@ impl SchemaCatalog {
         let secret_ref = Arc::new(secret);
 
         let old_secret = self.secret_by_id.get(&id).unwrap();
-        // check if secret name get updated.
-        if old_secret.name != name {
+        // check if the secret name gets updated.
+        if old_secret.name != name
+            && let Some(s) = self.secret_by_name.get(&old_secret.name)
+            && s.id == id
+        {
             self.secret_by_name.remove(&old_secret.name);
         }
 
@@ -557,6 +585,14 @@ impl SchemaCatalog {
     }
 
     /// Iterate all materialized views, excluding the indices.
+    pub fn iter_all_mvs(&self) -> impl Iterator<Item = &Arc<TableCatalog>> {
+        self.table_by_name
+            .iter()
+            .filter(|(_, v)| v.is_mview() && valid_table_name(&v.name))
+            .map(|(_, v)| v)
+    }
+
+    /// Iterate created materialized views, excluding the indices.
     pub fn iter_created_mvs(&self) -> impl Iterator<Item = &Arc<TableCatalog>> {
         self.table_by_name
             .iter()
diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_attribute.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_attribute.rs
index 89197e8f95dab..ef75cdd4af4ed 100644
--- a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_attribute.rs
+++ b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_attribute.rs
@@ -28,6 +28,10 @@ use risingwave_frontend_macro::system_catalog;
     "SELECT c.relation_id AS attrelid,
             c.name AS attname,
             c.type_oid AS atttypid,
+            CASE
+              WHEN c.udt_type = 'list' THEN 1::int2
+              ELSE 0::int2
+            END AS attndims,
             c.type_len AS attlen,
             c.position::smallint AS attnum,
             false AS attnotnull,
@@ -49,6 +53,7 @@ struct PgAttribute {
     attrelid: i32,
     attname: String,
     atttypid: i32,
+    attndims: i16,
     attlen: i16,
     attnum: i16,
     attnotnull: bool,
diff --git a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_class.rs b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_class.rs
index 537f09bdba4de..bfcc41bd8395d 100644
--- a/src/frontend/src/catalog/system_catalog/pg_catalog/pg_class.rs
+++ b/src/frontend/src/catalog/system_catalog/pg_catalog/pg_class.rs
@@ -27,6 +27,7 @@ use risingwave_frontend_macro::system_catalog;
         WHEN relation_type = 'view' THEN 'v'
         WHEN relation_type = 'materialized view' THEN 'm'
     END relkind,
+    0::int2 AS relpages,
     0 AS relam,
     0 AS reltablespace,
     ARRAY[]::varchar[] AS reloptions,
@@ -45,6 +46,7 @@ struct PgClass {
     // r = ordinary table, i = index, S = sequence, t = TOAST table, v = view, m = materialized view,
     // c = composite type, f = foreign table, p = partitioned table, I = partitioned index
     relkind: String,
+    relpages: i16,
     relam: i32,
     reltablespace: i32,
     reloptions: Vec<String>,
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_event_logs.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_event_logs.rs
index 1b0acfece2d01..ca5ad7efa3d53 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_event_logs.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_event_logs.rs
@@ -63,6 +63,7 @@ fn event_type(e: &Event) -> String {
         Event::CollectBarrierFail(_) => "COLLECT_BARRIER_FAIL",
         Event::WorkerNodePanic(_) => "WORKER_NODE_PANIC",
         Event::AutoSchemaChangeFail(_) => "AUTO_SCHEMA_CHANGE_FAIL",
+        Event::SinkFail(_) => "SINK_FAIL",
     }
     .into()
 }
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_fragments.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_fragments.rs
index ebe7369e46485..91f818e7919f7 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_fragments.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_fragments.rs
@@ -14,7 +14,6 @@
 
 use risingwave_common::types::Fields;
 use risingwave_frontend_macro::system_catalog;
-use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType;
 use risingwave_pb::stream_plan::FragmentTypeFlag;
 
 use crate::catalog::system_catalog::SysCatalogReaderImpl;
@@ -53,36 +52,27 @@ async fn read_rw_fragment(reader: &SysCatalogReaderImpl) -> Result<Vec<RwFragmen
 
     Ok(distributions
         .into_iter()
-        .map(|distribution| {
-            let distribution_type = distribution.distribution_type();
-            let max_parallelism = match distribution_type {
-                FragmentDistributionType::Single => 1,
-                FragmentDistributionType::Hash => distribution.vnode_count as i32,
-                FragmentDistributionType::Unspecified => unreachable!(),
-            };
-
-            RwFragment {
-                fragment_id: distribution.fragment_id as i32,
-                table_id: distribution.table_id as i32,
-                distribution_type: distribution.distribution_type().as_str_name().into(),
-                state_table_ids: distribution
-                    .state_table_ids
-                    .into_iter()
-                    .map(|id| id as i32)
-                    .collect(),
-                upstream_fragment_ids: distribution
-                    .upstream_fragment_ids
-                    .into_iter()
-                    .map(|id| id as i32)
-                    .collect(),
-                flags: extract_fragment_type_flag(distribution.fragment_type_mask)
-                    .into_iter()
-                    .flat_map(|t| t.as_str_name().strip_prefix("FRAGMENT_TYPE_FLAG_"))
-                    .map(|s| s.into())
-                    .collect(),
-                parallelism: distribution.parallelism as i32,
-                max_parallelism,
-            }
+        .map(|distribution| RwFragment {
+            fragment_id: distribution.fragment_id as i32,
+            table_id: distribution.table_id as i32,
+            distribution_type: distribution.distribution_type().as_str_name().into(),
+            state_table_ids: distribution
+                .state_table_ids
+                .into_iter()
+                .map(|id| id as i32)
+                .collect(),
+            upstream_fragment_ids: distribution
+                .upstream_fragment_ids
+                .into_iter()
+                .map(|id| id as i32)
+                .collect(),
+            flags: extract_fragment_type_flag(distribution.fragment_type_mask)
+                .into_iter()
+                .flat_map(|t| t.as_str_name().strip_prefix("FRAGMENT_TYPE_FLAG_"))
+                .map(|s| s.into())
+                .collect(),
+            parallelism: distribution.parallelism as i32,
+            max_parallelism: distribution.vnode_count as i32,
         })
         .collect())
 }
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs
index c593b35b18f87..a0e8d98b24b69 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_materialized_views.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use risingwave_common::catalog::CreateType;
 use risingwave_common::types::{Fields, Timestamptz};
 use risingwave_frontend_macro::system_catalog;
 use risingwave_pb::user::grant_privilege::Object;
@@ -33,6 +34,7 @@ struct RwMaterializedView {
     created_at: Option<Timestamptz>,
     initialized_at_cluster_version: Option<String>,
     created_at_cluster_version: Option<String>,
+    background_ddl: bool,
 }
 
 #[system_catalog(table, "rw_catalog.rw_materialized_views")]
@@ -45,7 +47,7 @@ fn read_rw_materialized_views(reader: &SysCatalogReaderImpl) -> Result<Vec<RwMat
 
     Ok(schemas
         .flat_map(|schema| {
-            schema.iter_created_mvs().map(|table| RwMaterializedView {
+            schema.iter_all_mvs().map(|table| RwMaterializedView {
                 id: table.id.table_id as i32,
                 name: table.name().into(),
                 schema_id: schema.id() as i32,
@@ -62,6 +64,7 @@ fn read_rw_materialized_views(reader: &SysCatalogReaderImpl) -> Result<Vec<RwMat
                 created_at: table.created_at_epoch.map(|e| e.as_timestamptz()),
                 initialized_at_cluster_version: table.initialized_at_cluster_version.clone(),
                 created_at_cluster_version: table.created_at_cluster_version.clone(),
+                background_ddl: table.create_type == CreateType::Background,
             })
         })
         .collect())
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs
index 441ac99b48cda..40df3dfc3a849 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_sources.rs
@@ -40,6 +40,7 @@ struct RwSource {
     created_at: Option<Timestamptz>,
     initialized_at_cluster_version: Option<String>,
     created_at_cluster_version: Option<String>,
+    is_shared: bool,
 }
 
 #[system_catalog(table, "rw_catalog.rw_sources")]
@@ -83,6 +84,7 @@ fn read_rw_sources_info(reader: &SysCatalogReaderImpl) -> Result<Vec<RwSource>>
                 created_at: source.created_at_epoch.map(|e| e.as_timestamptz()),
                 initialized_at_cluster_version: source.initialized_at_cluster_version.clone(),
                 created_at_cluster_version: source.created_at_cluster_version.clone(),
+                is_shared: source.info.is_shared(),
             })
         })
         .collect())
diff --git a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_views.rs b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_views.rs
index 6ebf16ddd40ac..7c156d783a1c3 100644
--- a/src/frontend/src/catalog/system_catalog/rw_catalog/rw_views.rs
+++ b/src/frontend/src/catalog/system_catalog/rw_catalog/rw_views.rs
@@ -45,7 +45,7 @@ fn read_rw_view_info(reader: &SysCatalogReaderImpl) -> Result<Vec<RwView>> {
                 name: view.name().to_string(),
                 schema_id: schema.id() as i32,
                 owner: view.owner as i32,
-                definition: view.create_sql(),
+                definition: view.create_sql(schema.name()),
                 acl: get_acl_items(&Object::ViewId(view.id), false, &users, username_map),
             })
         })
diff --git a/src/frontend/src/catalog/table_catalog.rs b/src/frontend/src/catalog/table_catalog.rs
index 8c33e55a0b164..b533e6d956685 100644
--- a/src/frontend/src/catalog/table_catalog.rs
+++ b/src/frontend/src/catalog/table_catalog.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::assert_matches::assert_matches;
 use std::collections::{HashMap, HashSet};
 
 use fixedbitset::FixedBitSet;
@@ -20,7 +21,7 @@ use risingwave_common::catalog::{
     ColumnCatalog, ConflictBehavior, CreateType, Field, Schema, StreamJobStatus, TableDesc,
     TableId, TableVersionId,
 };
-use risingwave_common::hash::VnodeCountCompat;
+use risingwave_common::hash::{VnodeCount, VnodeCountCompat};
 use risingwave_common::util::epoch::Epoch;
 use risingwave_common::util::sort_util::ColumnOrder;
 use risingwave_pb::catalog::table::{OptionalAssociatedSourceId, PbTableType, PbTableVersion};
@@ -174,19 +175,13 @@ pub struct TableCatalog {
 
     /// Total vnode count of the table.
     ///
-    /// Can be unset if the catalog is generated by the frontend and not yet persisted. This is
-    /// because the vnode count of each fragment (then internal tables) is determined by the
-    /// meta service. See also [`StreamMaterialize::derive_table_catalog`] and
-    /// [`TableCatalogBuilder::build`].
-    ///
-    /// On the contrary, if this comes from a [`PbTable`], the field must be `Some` no matter
-    /// whether the table is created before or after the version we introduced variable vnode
-    /// count support. This is because we've already handled backward compatibility during
-    /// conversion.
+    /// Can be [`VnodeCount::Placeholder`] if the catalog is generated by the frontend and the
+    /// corresponding job is still in `Creating` status, in which case calling [`Self::vnode_count`]
+    /// will panic.
     ///
     /// [`StreamMaterialize::derive_table_catalog`]: crate::optimizer::plan_node::StreamMaterialize::derive_table_catalog
     /// [`TableCatalogBuilder::build`]: crate::optimizer::plan_node::utils::TableCatalogBuilder::build
-    pub vnode_count: Option<usize>,
+    pub vnode_count: VnodeCount,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
@@ -408,8 +403,7 @@ impl TableCatalog {
     ///
     /// Panics if it's called on an incomplete (and not yet persisted) table catalog.
     pub fn vnode_count(&self) -> usize {
-        self.vnode_count
-            .expect("vnode count unset, called on an incomplete table catalog?")
+        self.vnode_count.value()
     }
 
     pub fn to_prost(&self, schema_id: SchemaId, database_id: DatabaseId) -> PbTable {
@@ -457,7 +451,7 @@ impl TableCatalog {
             initialized_at_cluster_version: self.initialized_at_cluster_version.clone(),
             retention_seconds: self.retention_seconds,
             cdc_table_id: self.cdc_table_id.clone(),
-            maybe_vnode_count: self.vnode_count.map(|v| v as _),
+            maybe_vnode_count: self.vnode_count.to_protobuf(),
         }
     }
 
@@ -563,7 +557,13 @@ impl From<PbTable> for TableCatalog {
             OptionalAssociatedSourceId::AssociatedSourceId(id) => id,
         });
         let name = tb.name.clone();
-        let vnode_count = tb.vnode_count();
+
+        let vnode_count = tb.vnode_count_inner();
+        if let VnodeCount::Placeholder = vnode_count {
+            // Only allow placeholder vnode count for creating tables.
+            // After the table is created, an `Update` notification will be used to update the vnode count field.
+            assert_matches!(stream_job_status, PbStreamJobStatus::Creating);
+        }
 
         let mut col_names = HashSet::new();
         let mut col_index: HashMap<i32, usize> = HashMap::new();
@@ -634,7 +634,7 @@ impl From<PbTable> for TableCatalog {
                 .map(TableId::from)
                 .collect_vec(),
             cdc_table_id: tb.cdc_table_id,
-            vnode_count: Some(vnode_count), /* from existing (persisted) tables, vnode_count must be set */
+            vnode_count,
         }
     }
 }
@@ -725,7 +725,7 @@ mod tests {
             initialized_at_cluster_version: None,
             version_column_index: None,
             cdc_table_id: None,
-            maybe_vnode_count: Some(233),
+            maybe_vnode_count: VnodeCount::set(233).to_protobuf(),
         }
         .into();
 
@@ -789,7 +789,7 @@ mod tests {
                 dependent_relations: vec![],
                 version_column_index: None,
                 cdc_table_id: None,
-                vnode_count: Some(233),
+                vnode_count: VnodeCount::set(233),
             }
         );
         assert_eq!(table, TableCatalog::from(table.to_prost(0, 0)));
diff --git a/src/frontend/src/catalog/view_catalog.rs b/src/frontend/src/catalog/view_catalog.rs
index 331613be9415d..e60db583b870b 100644
--- a/src/frontend/src/catalog/view_catalog.rs
+++ b/src/frontend/src/catalog/view_catalog.rs
@@ -54,8 +54,12 @@ impl ViewCatalog {
     }
 
     /// Returns the SQL statement that can be used to create this view.
-    pub fn create_sql(&self) -> String {
-        format!("CREATE VIEW {} AS {}", self.name, self.sql)
+    pub fn create_sql(&self, schema: String) -> String {
+        if schema == "public" {
+            format!("CREATE VIEW {} AS {}", self.name, self.sql)
+        } else {
+            format!("CREATE VIEW {}.{} AS {}", schema, self.name, self.sql)
+        }
     }
 }
 
diff --git a/src/frontend/src/expr/table_function.rs b/src/frontend/src/expr/table_function.rs
index 5f22398cc5834..cee4188e75791 100644
--- a/src/frontend/src/expr/table_function.rs
+++ b/src/frontend/src/expr/table_function.rs
@@ -14,7 +14,10 @@
 
 use std::sync::{Arc, LazyLock};
 
+use anyhow::Context;
 use itertools::Itertools;
+use mysql_async::consts::ColumnType as MySqlColumnType;
+use mysql_async::prelude::*;
 use risingwave_common::array::arrow::IcebergArrowConvert;
 use risingwave_common::types::{DataType, ScalarImpl, StructType};
 use risingwave_connector::source::iceberg::{create_parquet_stream_builder, list_s3_directory};
@@ -22,8 +25,8 @@ pub use risingwave_pb::expr::table_function::PbType as TableFunctionType;
 use risingwave_pb::expr::PbTableFunction;
 use thiserror_ext::AsReport;
 use tokio::runtime::Runtime;
-use tokio_postgres;
 use tokio_postgres::types::Type as TokioPgType;
+use {mysql_async, tokio_postgres};
 
 use super::{infer_type, Expr, ExprImpl, ExprRewriter, Literal, RwResult};
 use crate::catalog::function_catalog::{FunctionCatalog, FunctionKind};
@@ -298,7 +301,7 @@ impl TableFunction {
                     tokio::spawn(async move {
                         if let Err(e) = connection.await {
                             tracing::error!(
-                                "postgres_query_executor: connection error: {:?}",
+                                "mysql_query_executor: connection error: {:?}",
                                 e.as_report()
                             );
                         }
@@ -350,6 +353,162 @@ impl TableFunction {
         }
     }
 
+    pub fn new_mysql_query(args: Vec<ExprImpl>) -> RwResult<Self> {
+        static MYSQL_ARGS_LEN: usize = 6;
+        let args = {
+            if args.len() != MYSQL_ARGS_LEN {
+                return Err(BindError("mysql_query function only accepts 6 arguments: mysql_query(hostname varchar, port varchar, username varchar, password varchar, database_name varchar, mysql_query varchar)".to_string()).into());
+            }
+            let mut cast_args = Vec::with_capacity(MYSQL_ARGS_LEN);
+            for arg in args {
+                let arg = arg.cast_implicit(DataType::Varchar)?;
+                cast_args.push(arg);
+            }
+            cast_args
+        };
+        let evaled_args = {
+            let mut evaled_args: Vec<String> = Vec::with_capacity(MYSQL_ARGS_LEN);
+            for arg in &args {
+                match arg.try_fold_const() {
+                    Some(Ok(value)) => {
+                        let Some(scalar) = value else {
+                            return Err(BindError(
+                                "mysql_query function does not accept null arguments".to_string(),
+                            )
+                            .into());
+                        };
+                        evaled_args.push(scalar.into_utf8().into());
+                    }
+                    Some(Err(err)) => {
+                        return Err(err);
+                    }
+                    None => {
+                        return Err(BindError(
+                            "mysql_query function only accepts constant arguments".to_string(),
+                        )
+                        .into());
+                    }
+                }
+            }
+            evaled_args
+        };
+
+        #[cfg(madsim)]
+        {
+            return Err(crate::error::ErrorCode::BindError(
+                "postgres_query can't be used in the madsim mode".to_string(),
+            )
+            .into());
+        }
+
+        #[cfg(not(madsim))]
+        {
+            let schema = tokio::task::block_in_place(|| {
+                RUNTIME.block_on(async {
+                    let database_opts: mysql_async::Opts = {
+                        let port = evaled_args[1]
+                            .parse::<u16>()
+                            .context("failed to parse port")?;
+                        mysql_async::OptsBuilder::default()
+                            .ip_or_hostname(evaled_args[0].clone())
+                            .tcp_port(port)
+                            .user(Some(evaled_args[2].clone()))
+                            .pass(Some(evaled_args[3].clone()))
+                            .db_name(Some(evaled_args[4].clone()))
+                            .into()
+                    };
+
+                    let pool = mysql_async::Pool::new(database_opts);
+                    let mut conn = pool
+                        .get_conn()
+                        .await
+                        .context("failed to connect to mysql in binder")?;
+
+                    let query = evaled_args[5].clone();
+                    let statement = conn
+                        .prep(query)
+                        .await
+                        .context("failed to prepare mysql_query in binder")?;
+
+                    let mut rw_types = vec![];
+                    #[allow(clippy::never_loop)]
+                    for column in statement.columns() {
+                        let name = column.name_str().to_string();
+                        let data_type = match column.column_type() {
+                            // Boolean types
+                            MySqlColumnType::MYSQL_TYPE_BIT if column.column_length() == 1 => {
+                                DataType::Boolean
+                            }
+
+                            // Numeric types
+                            // NOTE(kwannoel): Although `bool/boolean` is a synonym of TINY(1) in MySQL,
+                            // we treat it as Int16 here. It is better to be straightforward in our conversion.
+                            MySqlColumnType::MYSQL_TYPE_TINY => DataType::Int16,
+                            MySqlColumnType::MYSQL_TYPE_SHORT => DataType::Int16,
+                            MySqlColumnType::MYSQL_TYPE_INT24 => DataType::Int32,
+                            MySqlColumnType::MYSQL_TYPE_LONG => DataType::Int32,
+                            MySqlColumnType::MYSQL_TYPE_LONGLONG => DataType::Int64,
+                            MySqlColumnType::MYSQL_TYPE_FLOAT => DataType::Float32,
+                            MySqlColumnType::MYSQL_TYPE_DOUBLE => DataType::Float64,
+                            MySqlColumnType::MYSQL_TYPE_NEWDECIMAL => DataType::Decimal,
+                            MySqlColumnType::MYSQL_TYPE_DECIMAL => DataType::Decimal,
+
+                            // Date time types
+                            MySqlColumnType::MYSQL_TYPE_YEAR => DataType::Int32,
+                            MySqlColumnType::MYSQL_TYPE_DATE => DataType::Date,
+                            MySqlColumnType::MYSQL_TYPE_NEWDATE => DataType::Date,
+                            MySqlColumnType::MYSQL_TYPE_TIME => DataType::Time,
+                            MySqlColumnType::MYSQL_TYPE_TIME2 => DataType::Time,
+                            MySqlColumnType::MYSQL_TYPE_DATETIME => DataType::Timestamp,
+                            MySqlColumnType::MYSQL_TYPE_DATETIME2 => DataType::Timestamp,
+                            MySqlColumnType::MYSQL_TYPE_TIMESTAMP => DataType::Timestamptz,
+                            MySqlColumnType::MYSQL_TYPE_TIMESTAMP2 => DataType::Timestamptz,
+
+                            // String types
+                            MySqlColumnType::MYSQL_TYPE_VARCHAR
+                            | MySqlColumnType::MYSQL_TYPE_STRING
+                            | MySqlColumnType::MYSQL_TYPE_VAR_STRING => DataType::Varchar,
+
+                            // JSON types
+                            MySqlColumnType::MYSQL_TYPE_JSON => DataType::Jsonb,
+
+                            // Binary types
+                            MySqlColumnType::MYSQL_TYPE_BIT
+                            | MySqlColumnType::MYSQL_TYPE_BLOB
+                            | MySqlColumnType::MYSQL_TYPE_TINY_BLOB
+                            | MySqlColumnType::MYSQL_TYPE_MEDIUM_BLOB
+                            | MySqlColumnType::MYSQL_TYPE_LONG_BLOB => DataType::Bytea,
+
+                            MySqlColumnType::MYSQL_TYPE_UNKNOWN
+                            | MySqlColumnType::MYSQL_TYPE_TYPED_ARRAY
+                            | MySqlColumnType::MYSQL_TYPE_ENUM
+                            | MySqlColumnType::MYSQL_TYPE_SET
+                            | MySqlColumnType::MYSQL_TYPE_GEOMETRY
+                            | MySqlColumnType::MYSQL_TYPE_NULL => {
+                                return Err(crate::error::ErrorCode::BindError(
+                                    format!("unsupported column type: {:?}", column.column_type())
+                                        .to_string(),
+                                )
+                                .into());
+                            }
+                        };
+                        rw_types.push((name, data_type));
+                    }
+                    Ok::<risingwave_common::types::DataType, anyhow::Error>(DataType::Struct(
+                        StructType::new(rw_types),
+                    ))
+                })
+            })?;
+
+            Ok(TableFunction {
+                args,
+                return_type: schema,
+                function_type: TableFunctionType::MysqlQuery,
+                user_defined: None,
+            })
+        }
+    }
+
     pub fn to_protobuf(&self) -> PbTableFunction {
         PbTableFunction {
             function_type: self.function_type as i32,
diff --git a/src/frontend/src/expr/type_inference/func.rs b/src/frontend/src/expr/type_inference/func.rs
index 99392ad87b971..719484b959f3a 100644
--- a/src/frontend/src/expr/type_inference/func.rs
+++ b/src/frontend/src/expr/type_inference/func.rs
@@ -1042,7 +1042,7 @@ fn narrow_same_type<'a>(
 }
 
 struct TypeDisplay<'a>(&'a Option<DataType>);
-impl<'a> std::fmt::Display for TypeDisplay<'a> {
+impl std::fmt::Display for TypeDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self.0 {
             Some(t) => t.fmt(f),
diff --git a/src/frontend/src/handler/alter_source_column.rs b/src/frontend/src/handler/alter_source_column.rs
index 013b27ef8a213..2d2e2c6698282 100644
--- a/src/frontend/src/handler/alter_source_column.rs
+++ b/src/frontend/src/handler/alter_source_column.rs
@@ -165,13 +165,31 @@ pub mod tests {
         let session = frontend.session_ref();
         let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
 
+        let sql = r#"create source s_shared (v1 int) with (
+            connector = 'kafka',
+            topic = 'abc',
+            properties.bootstrap.server = 'localhost:29092',
+        ) FORMAT PLAIN ENCODE JSON;"#;
+
+        frontend
+            .run_sql_with_session(session.clone(), sql)
+            .await
+            .unwrap();
+
+        frontend
+            .run_sql_with_session(session.clone(), "SET streaming_use_shared_source TO false;")
+            .await
+            .unwrap();
         let sql = r#"create source s (v1 int) with (
             connector = 'kafka',
             topic = 'abc',
             properties.bootstrap.server = 'localhost:29092',
           ) FORMAT PLAIN ENCODE JSON;"#;
 
-        frontend.run_sql(sql).await.unwrap();
+        frontend
+            .run_sql_with_session(session.clone(), sql)
+            .await
+            .unwrap();
 
         let get_source = || {
             let catalog_reader = session.env().catalog_reader().read_guard();
@@ -189,6 +207,8 @@ pub mod tests {
             .map(|col| (col.name(), (col.data_type().clone(), col.column_id())))
             .collect();
 
+        let sql = "alter source s_shared add column v2 varchar;";
+        assert_eq!("Feature is not yet implemented: alter shared source\nTracking issue: https://github.com/risingwavelabs/risingwave/issues/16003", &frontend.run_sql(sql).await.unwrap_err().to_string());
         let sql = "alter source s add column v2 varchar;";
         frontend.run_sql(sql).await.unwrap();
 
diff --git a/src/frontend/src/handler/alter_source_with_sr.rs b/src/frontend/src/handler/alter_source_with_sr.rs
index 17722625d5347..5e889ef9f0d7e 100644
--- a/src/frontend/src/handler/alter_source_with_sr.rs
+++ b/src/frontend/src/handler/alter_source_with_sr.rs
@@ -24,7 +24,7 @@ use risingwave_connector::WithPropertiesExt;
 use risingwave_pb::catalog::StreamSourceInfo;
 use risingwave_pb::plan_common::{EncodeType, FormatType};
 use risingwave_sqlparser::ast::{
-    CompatibleSourceSchema, ConnectorSchema, CreateSourceStatement, Encode, Format, ObjectName,
+    CompatibleFormatEncode, CreateSourceStatement, Encode, Format, FormatEncodeOptions, ObjectName,
     SqlOption, Statement,
 };
 use risingwave_sqlparser::parser::Parser;
@@ -120,7 +120,7 @@ pub fn fetch_source_catalog_with_db_schema_id(
 /// and if the FORMAT and ENCODE are modified.
 pub fn check_format_encode(
     original_source: &SourceCatalog,
-    new_connector_schema: &ConnectorSchema,
+    new_format_encode: &FormatEncodeOptions,
 ) -> Result<()> {
     let StreamSourceInfo {
         format, row_encode, ..
@@ -137,9 +137,7 @@ pub fn check_format_encode(
         .into());
     };
 
-    if new_connector_schema.format != old_format
-        || new_connector_schema.row_encode != old_row_encode
-    {
+    if new_format_encode.format != old_format || new_format_encode.row_encode != old_row_encode {
         bail_not_implemented!(
             "the original definition is FORMAT {:?} ENCODE {:?}, and altering them is not supported yet",
             &old_format,
@@ -153,19 +151,18 @@ pub fn check_format_encode(
 /// Refresh the source registry and get the added/dropped columns.
 pub async fn refresh_sr_and_get_columns_diff(
     original_source: &SourceCatalog,
-    connector_schema: &ConnectorSchema,
+    format_encode: &FormatEncodeOptions,
     session: &Arc<SessionImpl>,
 ) -> Result<(StreamSourceInfo, Vec<ColumnCatalog>, Vec<ColumnCatalog>)> {
     let mut with_properties = original_source.with_properties.clone();
-    validate_compatibility(connector_schema, &mut with_properties)?;
+    validate_compatibility(format_encode, &mut with_properties)?;
 
     if with_properties.is_cdc_connector() {
         bail_not_implemented!("altering a cdc source is not supported");
     }
 
     let (Some(columns_from_resolve_source), source_info) =
-        bind_columns_from_source(session, connector_schema, Either::Right(&with_properties))
-            .await?
+        bind_columns_from_source(session, format_encode, Either::Right(&with_properties)).await?
     else {
         // Source without schema registry is rejected.
         unreachable!("source without schema registry is rejected")
@@ -189,18 +186,18 @@ pub async fn refresh_sr_and_get_columns_diff(
     Ok((source_info, added_columns, dropped_columns))
 }
 
-fn get_connector_schema_from_source(source: &SourceCatalog) -> Result<ConnectorSchema> {
+fn get_format_encode_from_source(source: &SourceCatalog) -> Result<FormatEncodeOptions> {
     let [stmt]: [_; 1] = Parser::parse_sql(&source.definition)
         .context("unable to parse original source definition")?
         .try_into()
         .unwrap();
     let Statement::CreateSource {
-        stmt: CreateSourceStatement { source_schema, .. },
+        stmt: CreateSourceStatement { format_encode, .. },
     } = stmt
     else {
         unreachable!()
     };
-    Ok(source_schema.into_v2_with_warning())
+    Ok(format_encode.into_v2_with_warning())
 }
 
 pub async fn handler_refresh_schema(
@@ -208,14 +205,14 @@ pub async fn handler_refresh_schema(
     name: ObjectName,
 ) -> Result<RwPgResponse> {
     let (source, _, _) = fetch_source_catalog_with_db_schema_id(&handler_args.session, &name)?;
-    let connector_schema = get_connector_schema_from_source(&source)?;
-    handle_alter_source_with_sr(handler_args, name, connector_schema).await
+    let format_encode = get_format_encode_from_source(&source)?;
+    handle_alter_source_with_sr(handler_args, name, format_encode).await
 }
 
 pub async fn handle_alter_source_with_sr(
     handler_args: HandlerArgs,
     name: ObjectName,
-    connector_schema: ConnectorSchema,
+    format_encode: FormatEncodeOptions,
 ) -> Result<RwPgResponse> {
     let session = handler_args.session;
     let (source, database_id, schema_id) = fetch_source_catalog_with_db_schema_id(&session, &name)?;
@@ -232,9 +229,9 @@ pub async fn handle_alter_source_with_sr(
         bail_not_implemented!(issue = 16003, "alter shared source");
     }
 
-    check_format_encode(&source, &connector_schema)?;
+    check_format_encode(&source, &format_encode)?;
 
-    if !schema_has_schema_registry(&connector_schema) {
+    if !schema_has_schema_registry(&format_encode) {
         return Err(ErrorCode::NotSupported(
             "altering a source without schema registry".to_string(),
             "try `ALTER SOURCE .. ADD COLUMN ...` instead".to_string(),
@@ -243,7 +240,7 @@ pub async fn handle_alter_source_with_sr(
     }
 
     let (source_info, added_columns, dropped_columns) =
-        refresh_sr_and_get_columns_diff(&source, &connector_schema, &session).await?;
+        refresh_sr_and_get_columns_diff(&source, &format_encode, &session).await?;
 
     if !dropped_columns.is_empty() {
         bail_not_implemented!(
@@ -258,10 +255,10 @@ pub async fn handle_alter_source_with_sr(
     source.info = source_info;
     source.columns.extend(added_columns);
     source.definition =
-        alter_definition_format_encode(&source.definition, connector_schema.row_options.clone())?;
+        alter_definition_format_encode(&source.definition, format_encode.row_options.clone())?;
 
     let (format_encode_options, format_encode_secret_ref) = resolve_secret_ref_in_with_options(
-        WithOptions::try_from(connector_schema.row_options())?,
+        WithOptions::try_from(format_encode.row_options())?,
         session.as_ref(),
     )?
     .into_parts();
@@ -299,19 +296,19 @@ pub fn alter_definition_format_encode(
 
     match &mut stmt {
         Statement::CreateSource {
-            stmt: CreateSourceStatement { source_schema, .. },
+            stmt: CreateSourceStatement { format_encode, .. },
         }
         | Statement::CreateTable {
-            source_schema: Some(source_schema),
+            format_encode: Some(format_encode),
             ..
         } => {
-            match source_schema {
-                CompatibleSourceSchema::V2(schema) => {
+            match format_encode {
+                CompatibleFormatEncode::V2(schema) => {
                     schema.row_options = format_encode_options;
                 }
                 // TODO: Confirm the behavior of legacy source schema.
                 // Legacy source schema should be rejected by the handler and never reaches here.
-                CompatibleSourceSchema::RowFormat(_schema) => unreachable!(),
+                CompatibleFormatEncode::RowFormat(_schema) => unreachable!(),
             }
         }
         _ => unreachable!(),
@@ -348,7 +345,14 @@ pub mod tests {
         let session = frontend.session_ref();
         let schema_path = SchemaPath::Name(DEFAULT_SCHEMA_NAME);
 
-        frontend.run_sql(sql).await.unwrap();
+        frontend
+            .run_sql_with_session(session.clone(), "SET streaming_use_shared_source TO false;")
+            .await
+            .unwrap();
+        frontend
+            .run_sql_with_session(session.clone(), sql)
+            .await
+            .unwrap();
 
         let get_source = || {
             let catalog_reader = session.env().catalog_reader().read_guard();
diff --git a/src/frontend/src/handler/alter_swap_rename.rs b/src/frontend/src/handler/alter_swap_rename.rs
new file mode 100644
index 0000000000000..a1d23484576f8
--- /dev/null
+++ b/src/frontend/src/handler/alter_swap_rename.rs
@@ -0,0 +1,179 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::sync::Arc;
+
+use pgwire::pg_response::StatementType;
+use risingwave_common::bail_not_implemented;
+use risingwave_pb::ddl_service::alter_swap_rename_request;
+use risingwave_pb::ddl_service::alter_swap_rename_request::ObjectNameSwapPair;
+use risingwave_sqlparser::ast::ObjectName;
+
+use crate::catalog::root_catalog::SchemaPath;
+use crate::catalog::CatalogError;
+use crate::error::{ErrorCode, Result};
+use crate::handler::{HandlerArgs, RwPgResponse};
+use crate::session::SessionImpl;
+use crate::user::UserId;
+use crate::Binder;
+
+/// Check if the session user has the privilege to swap and rename the objects.
+fn check_swap_rename_privilege(
+    session_impl: &Arc<SessionImpl>,
+    src_owner: UserId,
+    target_owner: UserId,
+) -> Result<()> {
+    if !session_impl.is_super_user()
+        && (src_owner != session_impl.user_id() || target_owner != session_impl.user_id())
+    {
+        return Err(ErrorCode::PermissionDenied(format!(
+            "{} is not super user and not the owner of the objects.",
+            session_impl.user_name()
+        ))
+        .into());
+    }
+    Ok(())
+}
+
+pub async fn handle_swap_rename(
+    handler_args: HandlerArgs,
+    source_object: ObjectName,
+    target_object: ObjectName,
+    stmt_type: StatementType,
+) -> Result<RwPgResponse> {
+    let session = handler_args.session;
+    let db_name = session.database();
+    let (src_schema_name, src_obj_name) =
+        Binder::resolve_schema_qualified_name(db_name, source_object)?;
+    let search_path = session.config().search_path();
+    let user_name = &session.auth_context().user_name;
+    let src_schema_path = SchemaPath::new(src_schema_name.as_deref(), &search_path, user_name);
+    let (target_schema_name, target_obj_name) =
+        Binder::resolve_schema_qualified_name(db_name, target_object)?;
+    let target_schema_path =
+        SchemaPath::new(target_schema_name.as_deref(), &search_path, user_name);
+
+    let obj = match stmt_type {
+        StatementType::ALTER_SCHEMA => {
+            // TODO: support it until resolves https://github.com/risingwavelabs/risingwave/issues/19028
+            bail_not_implemented!("ALTER SCHEMA SWAP WITH is not supported yet");
+        }
+        StatementType::ALTER_TABLE | StatementType::ALTER_MATERIALIZED_VIEW => {
+            let catalog_reader = session.env().catalog_reader().read_guard();
+            let (src_table, _) = catalog_reader.get_created_table_by_name(
+                db_name,
+                src_schema_path,
+                &src_obj_name,
+            )?;
+            let (target_table, _) = catalog_reader.get_created_table_by_name(
+                db_name,
+                target_schema_path,
+                &target_obj_name,
+            )?;
+
+            if src_table.table_type != target_table.table_type {
+                return Err(ErrorCode::PermissionDenied(format!(
+                    "cannot swap between {} and {}: type mismatch",
+                    src_obj_name, target_obj_name
+                ))
+                .into());
+            }
+            if stmt_type == StatementType::ALTER_TABLE && !src_table.is_table() {
+                return Err(CatalogError::NotFound("table", src_obj_name.to_string()).into());
+            } else if stmt_type == StatementType::ALTER_MATERIALIZED_VIEW && !src_table.is_mview() {
+                return Err(
+                    CatalogError::NotFound("materialized view", src_obj_name.to_string()).into(),
+                );
+            }
+
+            check_swap_rename_privilege(&session, src_table.owner, target_table.owner)?;
+
+            alter_swap_rename_request::Object::Table(ObjectNameSwapPair {
+                src_object_id: src_table.id.table_id,
+                dst_object_id: target_table.id.table_id,
+            })
+        }
+        StatementType::ALTER_VIEW => {
+            let catalog_reader = session.env().catalog_reader().read_guard();
+            let (src_view, _) =
+                catalog_reader.get_view_by_name(db_name, src_schema_path, &src_obj_name)?;
+            let (target_view, _) =
+                catalog_reader.get_view_by_name(db_name, target_schema_path, &target_obj_name)?;
+            check_swap_rename_privilege(&session, src_view.owner, target_view.owner)?;
+
+            alter_swap_rename_request::Object::View(ObjectNameSwapPair {
+                src_object_id: src_view.id,
+                dst_object_id: target_view.id,
+            })
+        }
+        StatementType::ALTER_SOURCE => {
+            let catalog_reader = session.env().catalog_reader().read_guard();
+            let (src_source, _) =
+                catalog_reader.get_source_by_name(db_name, src_schema_path, &src_obj_name)?;
+            let (target_source, _) =
+                catalog_reader.get_source_by_name(db_name, target_schema_path, &target_obj_name)?;
+            check_swap_rename_privilege(&session, src_source.owner, target_source.owner)?;
+
+            alter_swap_rename_request::Object::Source(ObjectNameSwapPair {
+                src_object_id: src_source.id,
+                dst_object_id: target_source.id,
+            })
+        }
+        StatementType::ALTER_SINK => {
+            let catalog_reader = session.env().catalog_reader().read_guard();
+            let (src_sink, _) =
+                catalog_reader.get_sink_by_name(db_name, src_schema_path, &src_obj_name)?;
+            let (target_sink, _) =
+                catalog_reader.get_sink_by_name(db_name, target_schema_path, &target_obj_name)?;
+            check_swap_rename_privilege(
+                &session,
+                src_sink.owner.user_id,
+                target_sink.owner.user_id,
+            )?;
+
+            alter_swap_rename_request::Object::Sink(ObjectNameSwapPair {
+                src_object_id: src_sink.id.sink_id,
+                dst_object_id: target_sink.id.sink_id,
+            })
+        }
+        StatementType::ALTER_SUBSCRIPTION => {
+            let catalog_reader = session.env().catalog_reader().read_guard();
+            let (src_subscription, _) =
+                catalog_reader.get_subscription_by_name(db_name, src_schema_path, &src_obj_name)?;
+            let (target_subscription, _) = catalog_reader.get_subscription_by_name(
+                db_name,
+                target_schema_path,
+                &target_obj_name,
+            )?;
+            check_swap_rename_privilege(
+                &session,
+                src_subscription.owner.user_id,
+                target_subscription.owner.user_id,
+            )?;
+
+            alter_swap_rename_request::Object::Subscription(ObjectNameSwapPair {
+                src_object_id: src_subscription.id.subscription_id,
+                dst_object_id: target_subscription.id.subscription_id,
+            })
+        }
+        _ => {
+            unreachable!("handle_swap_rename: unsupported statement type")
+        }
+    };
+
+    let catalog_writer = session.catalog_writer()?;
+    catalog_writer.alter_swap_rename(obj).await?;
+
+    Ok(RwPgResponse::empty_result(stmt_type))
+}
diff --git a/src/frontend/src/handler/alter_table_column.rs b/src/frontend/src/handler/alter_table_column.rs
index e9ecb5713cb10..ce90fa94253f3 100644
--- a/src/frontend/src/handler/alter_table_column.rs
+++ b/src/frontend/src/handler/alter_table_column.rs
@@ -19,6 +19,7 @@ use anyhow::{anyhow, Context};
 use itertools::Itertools;
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::catalog::ColumnCatalog;
+use risingwave_common::hash::VnodeCount;
 use risingwave_common::types::DataType;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::{bail, bail_not_implemented};
@@ -28,13 +29,13 @@ use risingwave_pb::ddl_service::TableJobType;
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
 use risingwave_pb::stream_plan::{ProjectNode, StreamFragmentGraph};
 use risingwave_sqlparser::ast::{
-    AlterTableOperation, ColumnDef, ColumnOption, ConnectorSchema, DataType as AstDataType, Encode,
-    ObjectName, Statement, StructField,
+    AlterTableOperation, ColumnDef, ColumnOption, DataType as AstDataType, Encode,
+    FormatEncodeOptions, ObjectName, Statement, StructField,
 };
 use risingwave_sqlparser::parser::Parser;
 
 use super::create_source::get_json_schema_location;
-use super::create_table::{generate_stream_graph_for_table, ColumnIdGenerator};
+use super::create_table::{generate_stream_graph_for_replace_table, ColumnIdGenerator};
 use super::util::SourceSchemaCompatExt;
 use super::{HandlerArgs, RwPgResponse};
 use crate::catalog::root_catalog::SchemaPath;
@@ -50,14 +51,14 @@ pub async fn replace_table_with_definition(
     table_name: ObjectName,
     definition: Statement,
     original_catalog: &Arc<TableCatalog>,
-    source_schema: Option<ConnectorSchema>,
+    format_encode: Option<FormatEncodeOptions>,
 ) -> Result<()> {
     let (source, table, graph, col_index_mapping, job_type) = get_replace_table_plan(
         session,
         table_name,
         definition,
         original_catalog,
-        source_schema,
+        format_encode,
         None,
     )
     .await?;
@@ -85,7 +86,7 @@ pub async fn get_new_table_definition_for_cdc_table(
         .unwrap();
     let Statement::CreateTable {
         columns: original_columns,
-        source_schema,
+        format_encode,
         ..
     } = &mut definition
     else {
@@ -93,7 +94,7 @@ pub async fn get_new_table_definition_for_cdc_table(
     };
 
     assert!(
-        source_schema.is_none(),
+        format_encode.is_none(),
         "source schema should be None for CDC table"
     );
 
@@ -164,7 +165,7 @@ pub async fn get_replace_table_plan(
     table_name: ObjectName,
     definition: Statement,
     original_catalog: &Arc<TableCatalog>,
-    source_schema: Option<ConnectorSchema>,
+    format_encode: Option<FormatEncodeOptions>,
     new_version_columns: Option<Vec<ColumnCatalog>>, // only provided in auto schema change
 ) -> Result<(
     Option<Source>,
@@ -191,11 +192,11 @@ pub async fn get_replace_table_plan(
         panic!("unexpected statement type: {:?}", definition);
     };
 
-    let (mut graph, table, source, job_type) = generate_stream_graph_for_table(
+    let (mut graph, table, source, job_type) = generate_stream_graph_for_replace_table(
         session,
         table_name,
         original_catalog,
-        source_schema,
+        format_encode,
         handler_args.clone(),
         col_id_gen,
         columns.clone(),
@@ -244,7 +245,7 @@ pub async fn get_replace_table_plan(
     // Set some fields ourselves so that the meta service does not need to maintain them.
     let mut table = table;
     table.incoming_sinks = incoming_sink_ids.iter().copied().collect();
-    table.maybe_vnode_count = Some(original_catalog.vnode_count() as _);
+    table.maybe_vnode_count = VnodeCount::set(original_catalog.vnode_count()).to_protobuf();
 
     Ok((source, table, graph, col_index_mapping, job_type))
 }
@@ -325,19 +326,19 @@ pub async fn handle_alter_table_column(
         .unwrap();
     let Statement::CreateTable {
         columns,
-        source_schema,
+        format_encode,
         ..
     } = &mut definition
     else {
         panic!("unexpected statement: {:?}", definition);
     };
-    let source_schema = source_schema
+    let format_encode = format_encode
         .clone()
-        .map(|source_schema| source_schema.into_v2_with_warning());
+        .map(|format_encode| format_encode.into_v2_with_warning());
 
     let fail_if_has_schema_registry = || {
-        if let Some(source_schema) = &source_schema
-            && schema_has_schema_registry(source_schema)
+        if let Some(format_encode) = &format_encode
+            && schema_has_schema_registry(format_encode)
         {
             Err(ErrorCode::NotSupported(
                 "alter table with schema registry".to_string(),
@@ -459,14 +460,14 @@ pub async fn handle_alter_table_column(
         table_name,
         definition,
         &original_catalog,
-        source_schema,
+        format_encode,
     )
     .await?;
 
     Ok(PgResponse::empty_result(StatementType::ALTER_TABLE))
 }
 
-pub fn schema_has_schema_registry(schema: &ConnectorSchema) -> bool {
+pub fn schema_has_schema_registry(schema: &FormatEncodeOptions) -> bool {
     match schema.row_encode {
         Encode::Avro | Encode::Protobuf => true,
         Encode::Json => {
diff --git a/src/frontend/src/handler/alter_table_with_sr.rs b/src/frontend/src/handler/alter_table_with_sr.rs
index d932246759e22..b5489b28b58f8 100644
--- a/src/frontend/src/handler/alter_table_with_sr.rs
+++ b/src/frontend/src/handler/alter_table_with_sr.rs
@@ -16,7 +16,7 @@ use anyhow::{anyhow, Context};
 use fancy_regex::Regex;
 use pgwire::pg_response::StatementType;
 use risingwave_common::bail_not_implemented;
-use risingwave_sqlparser::ast::{ConnectorSchema, ObjectName, Statement};
+use risingwave_sqlparser::ast::{FormatEncodeOptions, ObjectName, Statement};
 use risingwave_sqlparser::parser::Parser;
 use thiserror_ext::AsReport;
 
@@ -29,15 +29,15 @@ use super::{HandlerArgs, RwPgResponse};
 use crate::error::{ErrorCode, Result};
 use crate::TableCatalog;
 
-fn get_connector_schema_from_table(table: &TableCatalog) -> Result<Option<ConnectorSchema>> {
+fn get_format_encode_from_table(table: &TableCatalog) -> Result<Option<FormatEncodeOptions>> {
     let [stmt]: [_; 1] = Parser::parse_sql(&table.definition)
         .context("unable to parse original table definition")?
         .try_into()
         .unwrap();
-    let Statement::CreateTable { source_schema, .. } = stmt else {
+    let Statement::CreateTable { format_encode, .. } = stmt else {
         unreachable!()
     };
-    Ok(source_schema.map(|schema| schema.into_v2_with_warning()))
+    Ok(format_encode.map(|schema| schema.into_v2_with_warning()))
 }
 
 pub async fn handle_refresh_schema(
@@ -51,9 +51,9 @@ pub async fn handle_refresh_schema(
         bail_not_implemented!("alter table with incoming sinks");
     }
 
-    let connector_schema = {
-        let connector_schema = get_connector_schema_from_table(&original_table)?;
-        if !connector_schema
+    let format_encode = {
+        let format_encode = get_format_encode_from_table(&original_table)?;
+        if !format_encode
             .as_ref()
             .is_some_and(schema_has_schema_registry)
         {
@@ -63,12 +63,12 @@ pub async fn handle_refresh_schema(
             )
             .into());
         }
-        connector_schema.unwrap()
+        format_encode.unwrap()
     };
 
     let definition = alter_definition_format_encode(
         &original_table.definition,
-        connector_schema.row_options.clone(),
+        format_encode.row_options.clone(),
     )?;
 
     let [definition]: [_; 1] = Parser::parse_sql(&definition)
@@ -81,7 +81,7 @@ pub async fn handle_refresh_schema(
         table_name,
         definition,
         &original_table,
-        Some(connector_schema),
+        Some(format_encode),
     )
     .await;
 
diff --git a/src/frontend/src/handler/create_connection.rs b/src/frontend/src/handler/create_connection.rs
index 987f0e9fdd897..d7ef3aa10b883 100644
--- a/src/frontend/src/handler/create_connection.rs
+++ b/src/frontend/src/handler/create_connection.rs
@@ -16,23 +16,16 @@ use std::collections::BTreeMap;
 
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_connector::source::kafka::PRIVATELINK_CONNECTION;
-use risingwave_pb::catalog::connection::private_link_service::PrivateLinkProvider;
 use risingwave_pb::ddl_service::create_connection_request;
 use risingwave_sqlparser::ast::CreateConnectionStatement;
 
 use super::RwPgResponse;
 use crate::binder::Binder;
 use crate::error::ErrorCode::ProtocolError;
-use crate::error::{Result, RwError};
+use crate::error::{ErrorCode, Result, RwError};
 use crate::handler::HandlerArgs;
 
 pub(crate) const CONNECTION_TYPE_PROP: &str = "type";
-pub(crate) const CONNECTION_PROVIDER_PROP: &str = "provider";
-pub(crate) const CONNECTION_SERVICE_NAME_PROP: &str = "service.name";
-pub(crate) const CONNECTION_TAGS_PROP: &str = "tags";
-
-pub(crate) const CLOUD_PROVIDER_MOCK: &str = "mock"; // fake privatelink provider for testing
-pub(crate) const CLOUD_PROVIDER_AWS: &str = "aws";
 
 #[inline(always)]
 fn get_connection_property_required(
@@ -48,58 +41,19 @@ fn get_connection_property_required(
             )))
         })
 }
-
-fn resolve_private_link_properties(
-    with_properties: &BTreeMap<String, String>,
-) -> Result<create_connection_request::PrivateLink> {
-    let provider =
-        match get_connection_property_required(with_properties, CONNECTION_PROVIDER_PROP)?.as_str()
-        {
-            CLOUD_PROVIDER_MOCK => PrivateLinkProvider::Mock,
-            CLOUD_PROVIDER_AWS => PrivateLinkProvider::Aws,
-            provider => {
-                return Err(RwError::from(ProtocolError(format!(
-                    "Unsupported privatelink provider {}",
-                    provider
-                ))));
-            }
-        };
-    match provider {
-        PrivateLinkProvider::Mock => Ok(create_connection_request::PrivateLink {
-            provider: provider.into(),
-            service_name: String::new(),
-            tags: None,
-        }),
-        PrivateLinkProvider::Aws => {
-            let service_name =
-                get_connection_property_required(with_properties, CONNECTION_SERVICE_NAME_PROP)?;
-            Ok(create_connection_request::PrivateLink {
-                provider: provider.into(),
-                service_name,
-                tags: with_properties.get(CONNECTION_TAGS_PROP).cloned(),
-            })
-        }
-        PrivateLinkProvider::Unspecified => Err(RwError::from(ProtocolError(
-            "Privatelink provider unspecified".to_string(),
-        ))),
-    }
-}
-
 fn resolve_create_connection_payload(
     with_properties: &BTreeMap<String, String>,
 ) -> Result<create_connection_request::Payload> {
     let connection_type = get_connection_property_required(with_properties, CONNECTION_TYPE_PROP)?;
-    let create_connection_payload = match connection_type.as_str() {
-        PRIVATELINK_CONNECTION => create_connection_request::Payload::PrivateLink(
-            resolve_private_link_properties(with_properties)?,
-        ),
-        _ => {
-            return Err(RwError::from(ProtocolError(format!(
-                "Connection type \"{connection_type}\" is not supported"
-            ))));
-        }
-    };
-    Ok(create_connection_payload)
+    match connection_type.as_str() {
+        PRIVATELINK_CONNECTION => Err(RwError::from(ErrorCode::Deprecated(
+            "CREATE CONNECTION to Private Link".to_string(),
+            "RisingWave Cloud Portal (Please refer to the doc https://docs.risingwave.com/cloud/create-a-connection/)".to_string(),
+        ))),
+        _ => Err(RwError::from(ProtocolError(format!(
+            "Connection type \"{connection_type}\" is not supported"
+        )))),
+    }
 }
 
 pub async fn handle_create_connection(
diff --git a/src/frontend/src/handler/create_database.rs b/src/frontend/src/handler/create_database.rs
index c5f94b54684d7..9b215ac1d9783 100644
--- a/src/frontend/src/handler/create_database.rs
+++ b/src/frontend/src/handler/create_database.rs
@@ -26,6 +26,7 @@ pub async fn handle_create_database(
     handler_args: HandlerArgs,
     database_name: ObjectName,
     if_not_exist: bool,
+    owner: Option<ObjectName>,
 ) -> Result<RwPgResponse> {
     let session = handler_args.session;
     let database_name = Binder::resolve_database_name(database_name)?;
@@ -57,9 +58,22 @@ pub async fn handle_create_database(
         }
     }
 
+    let database_owner = if let Some(owner) = owner {
+        let owner = Binder::resolve_user_name(owner)?;
+        session
+            .env()
+            .user_info_reader()
+            .read_guard()
+            .get_user_by_name(&owner)
+            .map(|u| u.id)
+            .ok_or_else(|| CatalogError::NotFound("user", owner.to_string()))?
+    } else {
+        session.user_id()
+    };
+
     let catalog_writer = session.catalog_writer()?;
     catalog_writer
-        .create_database(&database_name, session.user_id())
+        .create_database(&database_name, database_owner)
         .await?;
 
     Ok(PgResponse::empty_result(StatementType::CREATE_DATABASE))
diff --git a/src/frontend/src/handler/create_index.rs b/src/frontend/src/handler/create_index.rs
index 4b32bedc01ea2..1bf7ad16e1c93 100644
--- a/src/frontend/src/handler/create_index.rs
+++ b/src/frontend/src/handler/create_index.rs
@@ -21,11 +21,9 @@ use either::Either;
 use fixedbitset::FixedBitSet;
 use itertools::Itertools;
 use pgwire::pg_response::{PgResponse, StatementType};
-use risingwave_common::acl::AclMode;
 use risingwave_common::catalog::{IndexId, TableDesc, TableId};
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_pb::catalog::{PbIndex, PbIndexColumnProperties, PbStreamJobStatus, PbTable};
-use risingwave_pb::user::grant_privilege::Object;
 use risingwave_sqlparser::ast;
 use risingwave_sqlparser::ast::{Ident, ObjectName, OrderByExpr};
 
@@ -34,7 +32,6 @@ use crate::binder::Binder;
 use crate::catalog::root_catalog::SchemaPath;
 use crate::error::{ErrorCode, Result};
 use crate::expr::{Expr, ExprImpl, ExprRewriter, InputRef};
-use crate::handler::privilege::ObjectCheckItem;
 use crate::handler::HandlerArgs;
 use crate::optimizer::plan_expr_rewriter::ConstEvalRewriter;
 use crate::optimizer::plan_node::{Explain, LogicalProject, LogicalScan, StreamMaterialize};
@@ -83,11 +80,11 @@ pub(crate) fn gen_create_index_plan(
         );
     }
 
-    session.check_privileges(&[ObjectCheckItem::new(
-        table.owner,
-        AclMode::Select,
-        Object::TableId(table.id.table_id),
-    )])?;
+    if !session.is_super_user() && session.user_id() != table.owner {
+        return Err(
+            ErrorCode::PermissionDenied(format!("must be owner of table {}", table.name)).into(),
+        );
+    }
 
     let mut binder = Binder::new_for_stream(session);
     binder.bind_table(Some(&schema_name), &table_name, None)?;
@@ -202,7 +199,7 @@ pub(crate) fn gen_create_index_plan(
         &index_columns_ordered_expr,
         &include_columns_expr,
         // We use the first index column as distributed key by default if users
-        // haven't specify the distributed by columns.
+        // haven't specified the distributed by columns.
         if distributed_columns_expr.is_empty() {
             1
         } else {
@@ -221,7 +218,7 @@ pub(crate) fn gen_create_index_plan(
         index_table_prost.retention_seconds = table.retention_seconds;
     }
 
-    index_table_prost.owner = session.user_id();
+    index_table_prost.owner = table.owner;
     index_table_prost.dependent_relations = vec![table.id.table_id];
 
     let index_columns_len = index_columns_ordered_expr.len() as u32;
diff --git a/src/frontend/src/handler/create_schema.rs b/src/frontend/src/handler/create_schema.rs
index 46972a2757e3e..a5b04beb91e1b 100644
--- a/src/frontend/src/handler/create_schema.rs
+++ b/src/frontend/src/handler/create_schema.rs
@@ -29,7 +29,7 @@ pub async fn handle_create_schema(
     handler_args: HandlerArgs,
     schema_name: ObjectName,
     if_not_exist: bool,
-    user_specified: Option<ObjectName>,
+    owner: Option<ObjectName>,
 ) -> Result<RwPgResponse> {
     let session = handler_args.session;
     let database_name = session.database();
@@ -63,15 +63,15 @@ pub async fn handle_create_schema(
         (db.id(), db.owner())
     };
 
-    let schema_owner = if let Some(user_specified) = user_specified {
-        let user_specified = Binder::resolve_user_name(user_specified)?;
+    let schema_owner = if let Some(owner) = owner {
+        let owner = Binder::resolve_user_name(owner)?;
         session
             .env()
             .user_info_reader()
             .read_guard()
-            .get_user_by_name(&user_specified)
+            .get_user_by_name(&owner)
             .map(|u| u.id)
-            .ok_or_else(|| CatalogError::NotFound("user", user_specified.to_string()))?
+            .ok_or_else(|| CatalogError::NotFound("user", owner.to_string()))?
     } else {
         session.user_id()
     };
diff --git a/src/frontend/src/handler/create_sink.rs b/src/frontend/src/handler/create_sink.rs
index 8d5d65004b6e8..5385e2b4957c1 100644
--- a/src/frontend/src/handler/create_sink.rs
+++ b/src/frontend/src/handler/create_sink.rs
@@ -22,9 +22,7 @@ use maplit::{convert_args, hashmap};
 use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_common::array::arrow::arrow_schema_iceberg::DataType as ArrowDataType;
 use risingwave_common::array::arrow::IcebergArrowConvert;
-use risingwave_common::catalog::{
-    ColumnCatalog, ConnectionId, DatabaseId, Schema, SchemaId, TableId, UserId,
-};
+use risingwave_common::catalog::{ColumnCatalog, DatabaseId, Schema, SchemaId, TableId, UserId};
 use risingwave_common::secret::LocalSecretManager;
 use risingwave_common::types::DataType;
 use risingwave_common::{bail, catalog};
@@ -38,7 +36,7 @@ use risingwave_pb::ddl_service::{ReplaceTablePlan, TableJobType};
 use risingwave_pb::stream_plan::stream_node::{NodeBody, PbNodeBody};
 use risingwave_pb::stream_plan::{MergeNode, StreamFragmentGraph, StreamNode};
 use risingwave_sqlparser::ast::{
-    ConnectorSchema, CreateSink, CreateSinkStatement, EmitMode, Encode, ExplainOptions, Format,
+    CreateSink, CreateSinkStatement, EmitMode, Encode, ExplainOptions, Format, FormatEncodeOptions,
     Query, Statement,
 };
 use risingwave_sqlparser::parser::Parser;
@@ -56,7 +54,7 @@ use crate::error::{ErrorCode, Result, RwError};
 use crate::expr::{rewrite_now_to_proctime, ExprImpl, InputRef};
 use crate::handler::alter_table_column::fetch_table_catalog_for_alter;
 use crate::handler::create_mv::parse_column_names;
-use crate::handler::create_table::{generate_stream_graph_for_table, ColumnIdGenerator};
+use crate::handler::create_table::{generate_stream_graph_for_replace_table, ColumnIdGenerator};
 use crate::handler::privilege::resolve_query_privileges;
 use crate::handler::util::SourceSchemaCompatExt;
 use crate::handler::HandlerArgs;
@@ -92,12 +90,7 @@ pub async fn gen_sink_plan(
 
     let mut with_options = handler_args.with_options.clone();
 
-    let connection_id = {
-        let conn_id =
-            resolve_privatelink_in_with_option(&mut with_options, &sink_schema_name, session)?;
-        conn_id.map(ConnectionId)
-    };
-
+    resolve_privatelink_in_with_option(&mut with_options)?;
     let mut resolved_with_options = resolve_secret_ref_in_with_options(with_options, session)?;
 
     let partition_info = get_partition_compute_info(&resolved_with_options).await?;
@@ -266,7 +259,7 @@ pub async fn gen_sink_plan(
         SchemaId::new(sink_schema_id),
         DatabaseId::new(sink_database_id),
         UserId::new(session.user_id()),
-        connection_id,
+        None, // deprecated: private link connection id
         dependent_relations.into_iter().collect_vec(),
     );
 
@@ -650,7 +643,7 @@ pub(crate) async fn reparse_table_for_sink(
         .unwrap();
     let Statement::CreateTable {
         name,
-        source_schema,
+        format_encode,
         ..
     } = &definition
     else {
@@ -658,9 +651,9 @@ pub(crate) async fn reparse_table_for_sink(
     };
 
     let table_name = name.clone();
-    let source_schema = source_schema
+    let format_encode = format_encode
         .clone()
-        .map(|source_schema| source_schema.into_v2_with_warning());
+        .map(|format_encode| format_encode.into_v2_with_warning());
 
     // Create handler args as if we're creating a new table with the altered definition.
     let handler_args = HandlerArgs::new(session.clone(), &definition, Arc::from(""))?;
@@ -679,11 +672,11 @@ pub(crate) async fn reparse_table_for_sink(
         panic!("unexpected statement type: {:?}", definition);
     };
 
-    let (graph, table, source, _) = generate_stream_graph_for_table(
+    let (graph, table, source, _) = generate_stream_graph_for_replace_table(
         session,
         table_name,
         table_catalog,
-        source_schema,
+        format_encode,
         handler_args,
         col_id_gen,
         columns,
@@ -814,7 +807,10 @@ pub(crate) fn derive_default_column_project_for_sink(
 /// Transforms the (format, encode, options) from sqlparser AST into an internal struct `SinkFormatDesc`.
 /// This is an analogy to (part of) [`crate::handler::create_source::bind_columns_from_source`]
 /// which transforms sqlparser AST `SourceSchemaV2` into `StreamSourceInfo`.
-fn bind_sink_format_desc(session: &SessionImpl, value: ConnectorSchema) -> Result<SinkFormatDesc> {
+fn bind_sink_format_desc(
+    session: &SessionImpl,
+    value: FormatEncodeOptions,
+) -> Result<SinkFormatDesc> {
     use risingwave_connector::sink::catalog::{SinkEncode, SinkFormat};
     use risingwave_connector::sink::encoder::TimestamptzHandlingMode;
     use risingwave_sqlparser::ast::{Encode as E, Format as F};
@@ -932,7 +928,7 @@ static CONNECTORS_COMPATIBLE_FORMATS: LazyLock<HashMap<String, HashMap<Format, V
         ))
     });
 
-pub fn validate_compatibility(connector: &str, format_desc: &ConnectorSchema) -> Result<()> {
+pub fn validate_compatibility(connector: &str, format_desc: &FormatEncodeOptions) -> Result<()> {
     let compatible_formats = CONNECTORS_COMPATIBLE_FORMATS
         .get(connector)
         .ok_or_else(|| {
diff --git a/src/frontend/src/handler/create_source.rs b/src/frontend/src/handler/create_source.rs
index f168f7d0de3f6..9c37799422a59 100644
--- a/src/frontend/src/handler/create_source.rs
+++ b/src/frontend/src/handler/create_source.rs
@@ -21,6 +21,7 @@ use either::Either;
 use itertools::Itertools;
 use maplit::{convert_args, hashmap};
 use pgwire::pg_response::{PgResponse, StatementType};
+use rand::Rng;
 use risingwave_common::array::arrow::{arrow_schema_iceberg, IcebergArrowConvert};
 use risingwave_common::bail_not_implemented;
 use risingwave_common::catalog::{
@@ -62,8 +63,8 @@ use risingwave_pb::catalog::{PbSchemaRegistryNameStrategy, StreamSourceInfo, Wat
 use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType;
 use risingwave_pb::plan_common::{EncodeType, FormatType};
 use risingwave_sqlparser::ast::{
-    get_delimiter, AstString, ColumnDef, ConnectorSchema, CreateSourceStatement, Encode, Format,
-    ObjectName, ProtobufSchema, SourceWatermark, TableConstraint,
+    get_delimiter, AstString, ColumnDef, CreateSourceStatement, Encode, Format,
+    FormatEncodeOptions, ObjectName, ProtobufSchema, SourceWatermark, TableConstraint,
 };
 use risingwave_sqlparser::parser::{IncludeOption, IncludeOptionItem};
 use thiserror_ext::AsReport;
@@ -162,9 +163,9 @@ async fn extract_avro_table_schema(
         if let risingwave_connector::parser::EncodingProperties::Avro(avro_props) =
             &parser_config.encoding_config
             && matches!(avro_props.schema_location, SchemaLocation::File { .. })
-            && !format_encode_options
+            && format_encode_options
                 .get("with_deprecated_file_header")
-                .is_some_and(|v| v == "true")
+                .is_none_or(|v| v != "true")
         {
             bail_not_implemented!(issue = 12871, "avro without schema registry");
         }
@@ -299,7 +300,7 @@ fn get_name_strategy_or_default(name_strategy: Option<AstString>) -> Result<Opti
 /// Other special columns like additional columns (`INCLUDE`), and `row_id` column are not included.
 pub(crate) async fn bind_columns_from_source(
     session: &SessionImpl,
-    source_schema: &ConnectorSchema,
+    format_encode: &FormatEncodeOptions,
     with_properties: Either<&WithOptions, &WithOptionsSecResolved>,
 ) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
     const MESSAGE_NAME_KEY: &str = "message";
@@ -313,7 +314,7 @@ pub(crate) async fn bind_columns_from_source(
 
     let is_kafka: bool = options_with_secret.is_kafka_connector();
     let (format_encode_options, format_encode_secret_refs) = resolve_secret_ref_in_with_options(
-        WithOptions::try_from(source_schema.row_options())?,
+        WithOptions::try_from(format_encode.row_options())?,
         session,
     )?
     .into_parts();
@@ -345,18 +346,18 @@ pub(crate) async fn bind_columns_from_source(
     }
 
     let mut stream_source_info = StreamSourceInfo {
-        format: format_to_prost(&source_schema.format) as i32,
-        row_encode: row_encode_to_prost(&source_schema.row_encode) as i32,
+        format: format_to_prost(&format_encode.format) as i32,
+        row_encode: row_encode_to_prost(&format_encode.row_encode) as i32,
         format_encode_options,
         format_encode_secret_refs,
         ..Default::default()
     };
 
-    if source_schema.format == Format::Debezium {
+    if format_encode.format == Format::Debezium {
         try_consume_string_from_options(&mut format_encode_options_to_consume, DEBEZIUM_IGNORE_KEY);
     }
 
-    let columns = match (&source_schema.format, &source_schema.row_encode) {
+    let columns = match (&format_encode.format, &format_encode.row_encode) {
         (Format::Native, Encode::Native)
         | (Format::Plain, Encode::Bytes)
         | (Format::DebeziumMongo, Encode::Json) => None,
@@ -476,7 +477,7 @@ pub(crate) async fn bind_columns_from_source(
             Encode::Json,
         ) => {
             if matches!(
-                source_schema.format,
+                format_encode.format,
                 Format::Plain | Format::Upsert | Format::Debezium
             ) {
                 // Parse the value but throw it away.
@@ -524,8 +525,8 @@ pub(crate) async fn bind_columns_from_source(
     if !format_encode_options_to_consume.is_empty() {
         let err_string = format!(
             "Get unknown format_encode_options for {:?} {:?}: {}",
-            source_schema.format,
-            source_schema.row_encode,
+            format_encode.format,
+            format_encode.row_encode,
             format_encode_options_to_consume
                 .keys()
                 .map(|k| k.to_string())
@@ -539,10 +540,10 @@ pub(crate) async fn bind_columns_from_source(
 
 fn bind_columns_from_source_for_cdc(
     session: &SessionImpl,
-    source_schema: &ConnectorSchema,
+    format_encode: &FormatEncodeOptions,
 ) -> Result<(Option<Vec<ColumnCatalog>>, StreamSourceInfo)> {
     let (format_encode_options, format_encode_secret_refs) = resolve_secret_ref_in_with_options(
-        WithOptions::try_from(source_schema.row_options())?,
+        WithOptions::try_from(format_encode.row_options())?,
         session,
     )?
     .into_parts();
@@ -553,7 +554,7 @@ fn bind_columns_from_source_for_cdc(
         format_encode_secret_refs.clone(),
     )?;
 
-    match (&source_schema.format, &source_schema.row_encode) {
+    match (&format_encode.format, &format_encode.row_encode) {
         (Format::Plain, Encode::Json) => (),
         (format, encoding) => {
             // Note: parser will also check this. Just be extra safe here
@@ -568,8 +569,8 @@ fn bind_columns_from_source_for_cdc(
     let schema_config = get_json_schema_location(&mut format_encode_options_to_consume)?;
 
     let stream_source_info = StreamSourceInfo {
-        format: format_to_prost(&source_schema.format) as i32,
-        row_encode: row_encode_to_prost(&source_schema.row_encode) as i32,
+        format: format_to_prost(&format_encode.format) as i32,
+        row_encode: row_encode_to_prost(&format_encode.row_encode) as i32,
         format_encode_options,
         use_schema_registry: json_schema_infer_use_schema_registry(&schema_config),
         cdc_source_job: true,
@@ -580,8 +581,8 @@ fn bind_columns_from_source_for_cdc(
     if !format_encode_options_to_consume.is_empty() {
         let err_string = format!(
             "Get unknown format_encode_options for {:?} {:?}: {}",
-            source_schema.format,
-            source_schema.row_encode,
+            format_encode.format,
+            format_encode.row_encode,
             format_encode_options_to_consume
                 .keys()
                 .map(|k| k.to_string())
@@ -596,7 +597,7 @@ fn bind_columns_from_source_for_cdc(
 // check the additional column compatibility with the format and encode
 fn check_additional_column_compatibility(
     column_def: &IncludeOptionItem,
-    source_schema: Option<&ConnectorSchema>,
+    format_encode: Option<&FormatEncodeOptions>,
 ) -> Result<()> {
     // only allow header column have inner field
     if column_def.inner_field.is_some()
@@ -612,7 +613,7 @@ fn check_additional_column_compatibility(
     }
 
     // Payload column only allowed when encode is JSON
-    if let Some(schema) = source_schema
+    if let Some(schema) = format_encode
         && column_def
             .column_type
             .real_value()
@@ -629,7 +630,7 @@ fn check_additional_column_compatibility(
 
 /// add connector-spec columns to the end of column catalog
 pub fn handle_addition_columns(
-    source_schema: Option<&ConnectorSchema>,
+    format_encode: Option<&FormatEncodeOptions>,
     with_properties: &BTreeMap<String, String>,
     mut additional_columns: IncludeOption,
     columns: &mut Vec<ColumnCatalog>,
@@ -646,14 +647,8 @@ pub fn handle_addition_columns(
         ))));
     }
 
-    let latest_col_id: ColumnId = columns
-        .iter()
-        .map(|col| col.column_desc.column_id)
-        .max()
-        .unwrap(); // there must be at least one column in the column catalog
-
     while let Some(item) = additional_columns.pop() {
-        check_additional_column_compatibility(&item, source_schema)?;
+        check_additional_column_compatibility(&item, format_encode)?;
 
         let data_type = item
             .header_inner_expect_type
@@ -667,7 +662,7 @@ pub fn handle_addition_columns(
             );
         }
         let col = build_additional_column_desc(
-            latest_col_id.next(),
+            ColumnId::placeholder(),
             connector_name.as_str(),
             item.column_type.real_value().as_str(),
             item.column_alias.map(|alias| alias.real_value()),
@@ -684,7 +679,7 @@ pub fn handle_addition_columns(
 
 /// Bind columns from both source and sql defined.
 pub(crate) fn bind_all_columns(
-    source_schema: &ConnectorSchema,
+    format_encode: &FormatEncodeOptions,
     cols_from_source: Option<Vec<ColumnCatalog>>,
     cols_from_sql: Vec<ColumnCatalog>,
     col_defs_from_sql: &[ColumnDef],
@@ -713,7 +708,7 @@ pub(crate) fn bind_all_columns(
             // TODO(yuhao): https://github.com/risingwavelabs/risingwave/issues/12209
             Err(RwError::from(ProtocolError(
                     format!("User-defined schema from SQL is not allowed with FORMAT {} ENCODE {}. \
-                    Please refer to https://www.risingwave.dev/docs/current/sql-create-source/ for more information.", source_schema.format, source_schema.row_encode))))
+                    Please refer to https://www.risingwave.dev/docs/current/sql-create-source/ for more information.", format_encode.format, format_encode.row_encode))))
         }
     } else {
         if wildcard_idx.is_some() {
@@ -722,14 +717,8 @@ pub(crate) fn bind_all_columns(
                 "Remove the wildcard or use a source with external schema".to_string(),
             )));
         }
-        // FIXME(yuhao): cols_from_sql should be None is no `()` is given.
-        if cols_from_sql.is_empty() {
-            return Err(RwError::from(ProtocolError(
-                "Schema definition is required, either from SQL or schema registry.".to_string(),
-            )));
-        }
         let non_generated_sql_defined_columns = non_generated_sql_columns(col_defs_from_sql);
-        match (&source_schema.format, &source_schema.row_encode) {
+        match (&format_encode.format, &format_encode.row_encode) {
             (Format::DebeziumMongo, Encode::Json) => {
                 let mut columns = vec![
                     ColumnCatalog {
@@ -829,7 +818,7 @@ example:
 /// Bind column from source. Add key column to table columns if necessary.
 /// Return `pk_names`.
 pub(crate) async fn bind_source_pk(
-    source_schema: &ConnectorSchema,
+    format_encode: &FormatEncodeOptions,
     source_info: &StreamSourceInfo,
     columns: &mut [ColumnCatalog],
     sql_defined_pk_names: Vec<String>,
@@ -861,7 +850,7 @@ pub(crate) async fn bind_source_pk(
         })
         .collect_vec();
 
-    let res = match (&source_schema.format, &source_schema.row_encode) {
+    let res = match (&format_encode.format, &format_encode.row_encode) {
         (Format::Native, Encode::Native) | (Format::None, Encode::None) | (Format::Plain, _) => {
             sql_defined_pk_names
         }
@@ -1161,7 +1150,7 @@ pub fn validate_license(connector: &str) -> Result<()> {
 }
 
 pub fn validate_compatibility(
-    source_schema: &ConnectorSchema,
+    format_encode: &FormatEncodeOptions,
     props: &mut BTreeMap<String, String>,
 ) -> Result<()> {
     let mut connector = props
@@ -1195,9 +1184,9 @@ pub fn validate_compatibility(
 
     validate_license(&connector)?;
     if connector != KAFKA_CONNECTOR {
-        let res = match (&source_schema.format, &source_schema.row_encode) {
+        let res = match (&format_encode.format, &format_encode.row_encode) {
             (Format::Plain, Encode::Protobuf) | (Format::Plain, Encode::Avro) => {
-                let mut options = WithOptions::try_from(source_schema.row_options())?;
+                let mut options = WithOptions::try_from(format_encode.row_options())?;
                 let (_, use_schema_registry) = get_schema_location(options.inner_mut())?;
                 use_schema_registry
             }
@@ -1213,17 +1202,17 @@ pub fn validate_compatibility(
     }
 
     let compatible_encodes = compatible_formats
-        .get(&source_schema.format)
+        .get(&format_encode.format)
         .ok_or_else(|| {
             RwError::from(ProtocolError(format!(
                 "connector {} does not support format {:?}",
-                connector, source_schema.format
+                connector, format_encode.format
             )))
         })?;
-    if !compatible_encodes.contains(&source_schema.row_encode) {
+    if !compatible_encodes.contains(&format_encode.row_encode) {
         return Err(RwError::from(ProtocolError(format!(
             "connector {} does not support format {:?} with encode {:?}",
-            connector, source_schema.format, source_schema.row_encode
+            connector, format_encode.format, format_encode.row_encode
         ))));
     }
 
@@ -1279,7 +1268,7 @@ pub fn validate_compatibility(
 ///
 /// One should only call this function after all properties of all columns are resolved, like
 /// generated column descriptors.
-pub(super) async fn check_source_schema(
+pub(super) async fn check_format_encode(
     props: &WithOptionsSecResolved,
     row_id_index: Option<usize>,
     columns: &[ColumnCatalog],
@@ -1436,11 +1425,11 @@ pub async fn check_iceberg_source(
 
 pub fn bind_connector_props(
     handler_args: &HandlerArgs,
-    source_schema: &ConnectorSchema,
+    format_encode: &FormatEncodeOptions,
     is_create_source: bool,
 ) -> Result<WithOptions> {
     let mut with_properties = handler_args.with_options.clone().into_connector_props();
-    validate_compatibility(source_schema, &mut with_properties)?;
+    validate_compatibility(format_encode, &mut with_properties)?;
     let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
 
     if !is_create_source && with_properties.is_shareable_only_cdc_connector() {
@@ -1477,6 +1466,14 @@ pub fn bind_connector_props(
                 .to_string(),
         );
     }
+    if with_properties.is_mysql_cdc_connector() {
+        // Generate a random server id for mysql cdc source if needed
+        // `server.id` (in the range from 1 to 2^32 - 1). This value MUST be unique across whole replication
+        // group (that is, different from any other server id being used by any master or slave)
+        with_properties
+            .entry("server.id".to_string())
+            .or_insert(rand::thread_rng().gen_range(1..u32::MAX).to_string());
+    }
     Ok(with_properties)
 }
 
@@ -1484,7 +1481,7 @@ pub fn bind_connector_props(
 pub async fn bind_create_source_or_table_with_connector(
     handler_args: HandlerArgs,
     full_name: ObjectName,
-    source_schema: ConnectorSchema,
+    format_encode: FormatEncodeOptions,
     with_properties: WithOptions,
     sql_columns_defs: &[ColumnDef],
     constraints: Vec<TableConstraint>,
@@ -1512,11 +1509,11 @@ pub async fn bind_create_source_or_table_with_connector(
         .into());
     }
     if is_create_source {
-        match source_schema.format {
+        match format_encode.format {
             Format::Upsert => {
                 return Err(ErrorCode::BindError(format!(
                     "can't CREATE SOURCE with FORMAT UPSERT\n\nHint: use CREATE TABLE instead\n\n{}",
-                    hint_upsert(&source_schema.row_encode)
+                    hint_upsert(&format_encode.row_encode)
                 ))
                 .into());
             }
@@ -1532,7 +1529,7 @@ pub async fn bind_create_source_or_table_with_connector(
     let columns_from_sql = bind_sql_columns(sql_columns_defs)?;
 
     let mut columns = bind_all_columns(
-        &source_schema,
+        &format_encode,
         columns_from_resolve_source,
         columns_from_sql,
         sql_columns_defs,
@@ -1541,12 +1538,19 @@ pub async fn bind_create_source_or_table_with_connector(
 
     // add additional columns before bind pk, because `format upsert` requires the key column
     handle_addition_columns(
-        Some(&source_schema),
+        Some(&format_encode),
         &with_properties,
         include_column_options,
         &mut columns,
         false,
     )?;
+
+    if columns.is_empty() {
+        return Err(RwError::from(ProtocolError(
+            "Schema definition is required, either from SQL or schema registry.".to_string(),
+        )));
+    }
+
     // compatible with the behavior that add a hidden column `_rw_kafka_timestamp` to each message from Kafka source
     if is_create_source {
         // must behind `handle_addition_columns`
@@ -1555,13 +1559,12 @@ pub async fn bind_create_source_or_table_with_connector(
 
     // resolve privatelink connection for Kafka
     let mut with_properties = with_properties;
-    let connection_id =
-        resolve_privatelink_in_with_option(&mut with_properties, &schema_name, session)?;
+    resolve_privatelink_in_with_option(&mut with_properties)?;
 
     let with_properties = resolve_secret_ref_in_with_options(with_properties, session)?;
 
     let pk_names = bind_source_pk(
-        &source_schema,
+        &format_encode,
         &source_info,
         &mut columns,
         sql_pk_names,
@@ -1610,7 +1613,7 @@ pub async fn bind_create_source_or_table_with_connector(
         sql_columns_defs.to_vec(),
         &pk_col_ids,
     )?;
-    check_source_schema(&with_properties, row_id_index, &columns).await?;
+    check_format_encode(&with_properties, row_id_index, &columns).await?;
 
     let definition = handler_args.normalized_sql.clone();
 
@@ -1632,7 +1635,7 @@ pub async fn bind_create_source_or_table_with_connector(
         watermark_descs,
         associated_table_id,
         definition,
-        connection_id,
+        connection_id: None, // deprecated: private link connection id
         created_at_epoch: None,
         initialized_at_epoch: None,
         version: INITIAL_SOURCE_VERSION_ID,
@@ -1664,18 +1667,23 @@ pub async fn handle_create_source(
         )));
     }
 
-    let source_schema = stmt.source_schema.into_v2_with_warning();
-    let with_properties = bind_connector_props(&handler_args, &source_schema, true)?;
+    let format_encode = stmt.format_encode.into_v2_with_warning();
+    let with_properties = bind_connector_props(&handler_args, &format_encode, true)?;
 
     let create_cdc_source_job = with_properties.is_shareable_cdc_connector();
     let is_shared = create_cdc_source_job
         || (with_properties.is_shareable_non_cdc_connector()
-            && session.config().enable_shared_source());
+            && session
+                .env()
+                .streaming_config()
+                .developer
+                .enable_shared_source
+            && session.config().streaming_use_shared_source());
 
     let (columns_from_resolve_source, mut source_info) = if create_cdc_source_job {
-        bind_columns_from_source_for_cdc(&session, &source_schema)?
+        bind_columns_from_source_for_cdc(&session, &format_encode)?
     } else {
-        bind_columns_from_source(&session, &source_schema, Either::Left(&with_properties)).await?
+        bind_columns_from_source(&session, &format_encode, Either::Left(&with_properties)).await?
     };
     if is_shared {
         // Note: this field should be called is_shared. Check field doc for more details.
@@ -1687,7 +1695,7 @@ pub async fn handle_create_source(
     let (source_catalog, database_id, schema_id) = bind_create_source_or_table_with_connector(
         handler_args.clone(),
         stmt.source_name,
-        source_schema,
+        format_encode,
         with_properties,
         &stmt.columns,
         stmt.constraints,
diff --git a/src/frontend/src/handler/create_table.rs b/src/frontend/src/handler/create_table.rs
index cac81d904d038..1e5dc489c1a0c 100644
--- a/src/frontend/src/handler/create_table.rs
+++ b/src/frontend/src/handler/create_table.rs
@@ -35,6 +35,7 @@ use risingwave_connector::source::cdc::external::{
     ExternalTableConfig, ExternalTableImpl, DATABASE_NAME_KEY, SCHEMA_NAME_KEY, TABLE_NAME_KEY,
 };
 use risingwave_connector::{source, WithOptionsSecResolved};
+use risingwave_pb::catalog::source::OptionalAssociatedTableId;
 use risingwave_pb::catalog::{PbSource, PbTable, Table, WatermarkDesc};
 use risingwave_pb::ddl_service::TableJobType;
 use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn;
@@ -43,8 +44,8 @@ use risingwave_pb::plan_common::{
 };
 use risingwave_pb::stream_plan::StreamFragmentGraph;
 use risingwave_sqlparser::ast::{
-    CdcTableInfo, ColumnDef, ColumnOption, ConnectorSchema, DataType as AstDataType,
-    ExplainOptions, Format, ObjectName, OnConflict, SourceWatermark, TableConstraint,
+    CdcTableInfo, ColumnDef, ColumnOption, DataType as AstDataType, ExplainOptions, Format,
+    FormatEncodeOptions, ObjectName, OnConflict, SourceWatermark, TableConstraint,
 };
 use risingwave_sqlparser::parser::IncludeOption;
 use thiserror_ext::AsReport;
@@ -468,7 +469,7 @@ pub(crate) async fn gen_create_table_plan_with_source(
     column_defs: Vec<ColumnDef>,
     wildcard_idx: Option<usize>,
     constraints: Vec<TableConstraint>,
-    source_schema: ConnectorSchema,
+    format_encode: FormatEncodeOptions,
     source_watermarks: Vec<SourceWatermark>,
     mut col_id_gen: ColumnIdGenerator,
     append_only: bool,
@@ -477,28 +478,28 @@ pub(crate) async fn gen_create_table_plan_with_source(
     include_column_options: IncludeOption,
 ) -> Result<(PlanRef, Option<PbSource>, PbTable)> {
     if append_only
-        && source_schema.format != Format::Plain
-        && source_schema.format != Format::Native
+        && format_encode.format != Format::Plain
+        && format_encode.format != Format::Native
     {
         return Err(ErrorCode::BindError(format!(
             "Append only table does not support format {}.",
-            source_schema.format
+            format_encode.format
         ))
         .into());
     }
 
     let session = &handler_args.session;
-    let with_properties = bind_connector_props(&handler_args, &source_schema, false)?;
+    let with_properties = bind_connector_props(&handler_args, &format_encode, false)?;
 
     let (columns_from_resolve_source, source_info) =
-        bind_columns_from_source(session, &source_schema, Either::Left(&with_properties)).await?;
+        bind_columns_from_source(session, &format_encode, Either::Left(&with_properties)).await?;
 
     let overwrite_options = OverwriteOptions::new(&mut handler_args);
     let rate_limit = overwrite_options.source_rate_limit;
     let (source_catalog, database_id, schema_id) = bind_create_source_or_table_with_connector(
         handler_args.clone(),
         table_name,
-        source_schema,
+        format_encode,
         with_properties,
         &column_defs,
         constraints,
@@ -707,7 +708,7 @@ fn gen_table_plan_inner(
             return Err(ErrorCode::InvalidInputSyntax(
                 "When PRIMARY KEY constraint applied to an APPEND ONLY table, the ON CONFLICT behavior must be DO NOTHING.".to_owned(),
             )
-            .into());
+                .into());
         }
         Some(on_conflict)
     } else {
@@ -885,18 +886,17 @@ fn derive_with_options_for_cdc_table(
 ) -> Result<WithOptionsSecResolved> {
     use source::cdc::{MYSQL_CDC_CONNECTOR, POSTGRES_CDC_CONNECTOR, SQL_SERVER_CDC_CONNECTOR};
     // we should remove the prefix from `full_table_name`
-    let mut connect_properties = source_with_properties.clone();
+    let mut with_options = source_with_properties.clone();
     if let Some(connector) = source_with_properties.get(UPSTREAM_SOURCE_KEY) {
-        let table_name = match connector.as_str() {
+        match connector.as_str() {
             MYSQL_CDC_CONNECTOR => {
-                let db_name = connect_properties.get(DATABASE_NAME_KEY).ok_or_else(|| {
-                    anyhow!("{} not found in source properties", DATABASE_NAME_KEY)
+                // MySQL doesn't allow '.' in database name and table name, so we can split the
+                // external table name by '.' to get the table name
+                let (db_name, table_name) = external_table_name.split_once('.').ok_or_else(|| {
+                    anyhow!("The upstream table name must contain database name prefix, e.g. 'database.table'")
                 })?;
-
-                let prefix = format!("{}.", db_name.as_str());
-                external_table_name
-                    .strip_prefix(prefix.as_str())
-                    .ok_or_else(|| anyhow!("The upstream table name must contain database name prefix, e.g. 'mydb.table'."))?
+                with_options.insert(DATABASE_NAME_KEY.into(), db_name.into());
+                with_options.insert(TABLE_NAME_KEY.into(), table_name.into());
             }
             POSTGRES_CDC_CONNECTOR => {
                 let (schema_name, table_name) = external_table_name
@@ -904,9 +904,8 @@ fn derive_with_options_for_cdc_table(
                     .ok_or_else(|| anyhow!("The upstream table name must contain schema name prefix, e.g. 'public.table'"))?;
 
                 // insert 'schema.name' into connect properties
-                connect_properties.insert(SCHEMA_NAME_KEY.into(), schema_name.into());
-
-                table_name
+                with_options.insert(SCHEMA_NAME_KEY.into(), schema_name.into());
+                with_options.insert(TABLE_NAME_KEY.into(), table_name.into());
             }
             SQL_SERVER_CDC_CONNECTOR => {
                 // SQL Server external table name is in 'databaseName.schemaName.tableName' pattern,
@@ -924,9 +923,8 @@ fn derive_with_options_for_cdc_table(
                     })?;
 
                 // insert 'schema.name' into connect properties
-                connect_properties.insert(SCHEMA_NAME_KEY.into(), schema_name.into());
-
-                table_name
+                with_options.insert(SCHEMA_NAME_KEY.into(), schema_name.into());
+                with_options.insert(TABLE_NAME_KEY.into(), table_name.into());
             }
             _ => {
                 return Err(RwError::from(anyhow!(
@@ -935,16 +933,15 @@ fn derive_with_options_for_cdc_table(
                 )));
             }
         };
-        connect_properties.insert(TABLE_NAME_KEY.into(), table_name.into());
     }
-    Ok(connect_properties)
+    Ok(with_options)
 }
 
 #[allow(clippy::too_many_arguments)]
 pub(super) async fn handle_create_table_plan(
     handler_args: HandlerArgs,
     explain_options: ExplainOptions,
-    source_schema: Option<ConnectorSchema>,
+    format_encode: Option<FormatEncodeOptions>,
     cdc_table_info: Option<CdcTableInfo>,
     table_name: ObjectName,
     column_defs: Vec<ColumnDef>,
@@ -957,16 +954,16 @@ pub(super) async fn handle_create_table_plan(
     include_column_options: IncludeOption,
 ) -> Result<(PlanRef, Option<PbSource>, PbTable, TableJobType)> {
     let col_id_gen = ColumnIdGenerator::new_initial();
-    let source_schema = check_create_table_with_source(
+    let format_encode = check_create_table_with_source(
         &handler_args.with_options,
-        source_schema,
+        format_encode,
         &include_column_options,
         &cdc_table_info,
     )?;
 
     let ((plan, source, table), job_type) =
-        match (source_schema, cdc_table_info.as_ref()) {
-            (Some(source_schema), None) => (
+        match (format_encode, cdc_table_info.as_ref()) {
+            (Some(format_encode), None) => (
                 gen_create_table_plan_with_source(
                     handler_args,
                     explain_options,
@@ -974,7 +971,7 @@ pub(super) async fn handle_create_table_plan(
                     column_defs,
                     wildcard_idx,
                     constraints,
-                    source_schema,
+                    format_encode,
                     source_watermarks,
                     col_id_gen,
                     append_only,
@@ -1019,12 +1016,12 @@ pub(super) async fn handle_create_table_plan(
                     session.get_database_and_schema_id_for_create(schema_name.clone())?;
 
                 // cdc table cannot be append-only
-                let (source_schema, source_name) =
+                let (format_encode, source_name) =
                     Binder::resolve_schema_qualified_name(db_name, cdc_table.source_name.clone())?;
 
                 let source = {
                     let catalog_reader = session.env().catalog_reader().read_guard();
-                    let schema_name = source_schema
+                    let schema_name = format_encode
                         .clone()
                         .unwrap_or(DEFAULT_SCHEMA_NAME.to_string());
                     let (source, _) = catalog_reader.get_source_by_name(
@@ -1034,7 +1031,7 @@ pub(super) async fn handle_create_table_plan(
                     )?;
                     source.clone()
                 };
-                let connect_properties = derive_with_options_for_cdc_table(
+                let cdc_with_options = derive_with_options_for_cdc_table(
                     &source.with_properties,
                     cdc_table.external_table_name.clone(),
                 )?;
@@ -1042,7 +1039,7 @@ pub(super) async fn handle_create_table_plan(
                 let (columns, pk_names) = derive_schema_for_cdc_table(
                     &column_defs,
                     &constraints,
-                    connect_properties.clone(),
+                    cdc_with_options.clone(),
                     wildcard_idx.is_some(),
                     None,
                 )
@@ -1057,7 +1054,7 @@ pub(super) async fn handle_create_table_plan(
                     column_defs,
                     columns,
                     pk_names,
-                    connect_properties,
+                    cdc_with_options,
                     col_id_gen,
                     on_conflict,
                     with_version_column,
@@ -1162,7 +1159,7 @@ struct CdcSchemaChangeArgs {
 async fn derive_schema_for_cdc_table(
     column_defs: &Vec<ColumnDef>,
     constraints: &Vec<TableConstraint>,
-    connect_properties: WithOptionsSecResolved,
+    cdc_with_options: WithOptionsSecResolved,
     need_auto_schema_map: bool,
     schema_change_args: Option<CdcSchemaChangeArgs>,
 ) -> Result<(Vec<ColumnCatalog>, Vec<String>)> {
@@ -1176,7 +1173,7 @@ async fn derive_schema_for_cdc_table(
                     "Please define the schema manually".to_owned(),
                 )
             })?;
-        let (options, secret_refs) = connect_properties.into_parts();
+        let (options, secret_refs) = cdc_with_options.into_parts();
         let config = ExternalTableConfig::try_from_btreemap(options, secret_refs)
             .context("failed to extract external table config")?;
 
@@ -1239,7 +1236,7 @@ pub async fn handle_create_table(
     wildcard_idx: Option<usize>,
     constraints: Vec<TableConstraint>,
     if_not_exists: bool,
-    source_schema: Option<ConnectorSchema>,
+    format_encode: Option<FormatEncodeOptions>,
     source_watermarks: Vec<SourceWatermark>,
     append_only: bool,
     on_conflict: Option<OnConflict>,
@@ -1267,7 +1264,7 @@ pub async fn handle_create_table(
         let (plan, source, table, job_type) = handle_create_table_plan(
             handler_args,
             ExplainOptions::default(),
-            source_schema,
+            format_encode,
             cdc_table_info,
             table_name.clone(),
             column_defs,
@@ -1302,13 +1299,13 @@ pub async fn handle_create_table(
 
 pub fn check_create_table_with_source(
     with_options: &WithOptions,
-    source_schema: Option<ConnectorSchema>,
+    format_encode: Option<FormatEncodeOptions>,
     include_column_options: &IncludeOption,
     cdc_table_info: &Option<CdcTableInfo>,
-) -> Result<Option<ConnectorSchema>> {
+) -> Result<Option<FormatEncodeOptions>> {
     // skip check for cdc table
     if cdc_table_info.is_some() {
-        return Ok(source_schema);
+        return Ok(format_encode);
     }
     let defined_source = with_options.contains_key(UPSTREAM_SOURCE_KEY);
     if !include_column_options.is_empty() && !defined_source {
@@ -1318,19 +1315,19 @@ pub fn check_create_table_with_source(
         .into());
     }
     if defined_source {
-        source_schema.as_ref().ok_or_else(|| {
+        format_encode.as_ref().ok_or_else(|| {
             ErrorCode::InvalidInputSyntax("Please specify a source schema using FORMAT".to_owned())
         })?;
     }
-    Ok(source_schema)
+    Ok(format_encode)
 }
 
 #[allow(clippy::too_many_arguments)]
-pub async fn generate_stream_graph_for_table(
+pub async fn generate_stream_graph_for_replace_table(
     _session: &Arc<SessionImpl>,
     table_name: ObjectName,
     original_catalog: &Arc<TableCatalog>,
-    source_schema: Option<ConnectorSchema>,
+    format_encode: Option<FormatEncodeOptions>,
     handler_args: HandlerArgs,
     col_id_gen: ColumnIdGenerator,
     column_defs: Vec<ColumnDef>,
@@ -1345,8 +1342,8 @@ pub async fn generate_stream_graph_for_table(
 ) -> Result<(StreamFragmentGraph, Table, Option<PbSource>, TableJobType)> {
     use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
 
-    let ((plan, source, table), job_type) = match (source_schema, cdc_table_info.as_ref()) {
-        (Some(source_schema), None) => (
+    let ((plan, mut source, table), job_type) = match (format_encode, cdc_table_info.as_ref()) {
+        (Some(format_encode), None) => (
             gen_create_table_plan_with_source(
                 handler_args,
                 ExplainOptions::default(),
@@ -1354,7 +1351,7 @@ pub async fn generate_stream_graph_for_table(
                 column_defs,
                 wildcard_idx,
                 constraints,
-                source_schema,
+                format_encode,
                 source_watermarks,
                 col_id_gen,
                 append_only,
@@ -1445,13 +1442,18 @@ pub async fn generate_stream_graph_for_table(
     let graph = build_graph(plan)?;
 
     // Fill the original table ID.
-    let table = Table {
+    let mut table = Table {
         id: original_catalog.id().table_id(),
-        optional_associated_source_id: original_catalog
-            .associated_source_id()
-            .map(|source_id| OptionalAssociatedSourceId::AssociatedSourceId(source_id.into())),
         ..table
     };
+    if let Some(source_id) = original_catalog.associated_source_id() {
+        table.optional_associated_source_id = Some(OptionalAssociatedSourceId::AssociatedSourceId(
+            source_id.table_id,
+        ));
+        source.as_mut().unwrap().id = source_id.table_id;
+        source.as_mut().unwrap().optional_associated_table_id =
+            Some(OptionalAssociatedTableId::AssociatedTableId(table.id))
+    }
 
     Ok((graph, table, source, job_type))
 }
@@ -1467,12 +1469,12 @@ fn get_source_and_resolved_table_name(
     let (database_id, schema_id) =
         session.get_database_and_schema_id_for_create(schema_name.clone())?;
 
-    let (source_schema, source_name) =
+    let (format_encode, source_name) =
         Binder::resolve_schema_qualified_name(db_name, cdc_table.source_name.clone())?;
 
     let source = {
         let catalog_reader = session.env().catalog_reader().read_guard();
-        let schema_name = source_schema.unwrap_or(DEFAULT_SCHEMA_NAME.to_string());
+        let schema_name = format_encode.unwrap_or(DEFAULT_SCHEMA_NAME.to_string());
         let (source, _) = catalog_reader.get_source_by_name(
             db_name,
             SchemaPath::Name(schema_name.as_str()),
diff --git a/src/frontend/src/handler/explain.rs b/src/frontend/src/handler/explain.rs
index ed22462a66888..1740c161c3fbe 100644
--- a/src/frontend/src/handler/explain.rs
+++ b/src/frontend/src/handler/explain.rs
@@ -16,7 +16,9 @@ use pgwire::pg_response::{PgResponse, StatementType};
 use risingwave_batch::worker_manager::worker_node_manager::WorkerNodeSelector;
 use risingwave_common::bail_not_implemented;
 use risingwave_common::types::Fields;
-use risingwave_sqlparser::ast::{ExplainOptions, ExplainType, Statement};
+use risingwave_sqlparser::ast::{
+    ExplainFormat, ExplainOptions, ExplainType, FetchCursorStatement, Statement,
+};
 use thiserror_ext::AsReport;
 
 use super::create_index::{gen_create_index_plan, resolve_index_schema};
@@ -55,7 +57,7 @@ async fn do_handle_explain(
                 name,
                 columns,
                 constraints,
-                source_schema,
+                format_encode,
                 source_watermarks,
                 append_only,
                 on_conflict,
@@ -65,12 +67,12 @@ async fn do_handle_explain(
                 wildcard_idx,
                 ..
             } => {
-                let source_schema = source_schema.map(|s| s.into_v2_with_warning());
+                let format_encode = format_encode.map(|s| s.into_v2_with_warning());
 
                 let (plan, _source, _table, _job_type) = handle_create_table_plan(
                     handler_args,
                     explain_options,
-                    source_schema,
+                    format_encode,
                     cdc_table_info,
                     name.clone(),
                     columns,
@@ -94,6 +96,18 @@ async fn do_handle_explain(
                 (Ok(plan), context)
             }
 
+            Statement::FetchCursor {
+                stmt: FetchCursorStatement { cursor_name, .. },
+            } => {
+                let cursor_manager = session.clone().get_cursor_manager();
+                let plan = cursor_manager
+                    .gen_batch_plan_with_subscription_cursor(cursor_name, handler_args)
+                    .await
+                    .map(|x| x.plan)?;
+                let context = plan.ctx();
+                (Ok(plan), context)
+            }
+
             // For other queries without `await` point, we can keep a copy of reference to the
             // `OptimizerContext` even if the planning fails. This enables us to log the partial
             // traces for better debugging experience.
@@ -179,6 +193,7 @@ async fn do_handle_explain(
         let explain_trace = context.is_explain_trace();
         let explain_verbose = context.is_explain_verbose();
         let explain_type = context.explain_type();
+        let explain_format = context.explain_format();
 
         if explain_trace {
             let trace = context.take_trace();
@@ -212,7 +227,10 @@ async fn do_handle_explain(
             ExplainType::Physical => {
                 // if explain trace is on, the plan has been in the rows
                 if !explain_trace && let Ok(plan) = &plan {
-                    blocks.push(plan.explain_to_string());
+                    match explain_format {
+                        ExplainFormat::Text => blocks.push(plan.explain_to_string()),
+                        ExplainFormat::Json => blocks.push(plan.explain_to_json()),
+                    }
                 }
             }
             ExplainType::Logical => {
@@ -248,6 +266,21 @@ pub async fn handle_explain(
     if analyze {
         bail_not_implemented!(issue = 4856, "explain analyze");
     }
+    if options.trace && options.explain_format == ExplainFormat::Json {
+        return Err(ErrorCode::NotSupported(
+            "EXPLAIN (TRACE, JSON FORMAT)".to_string(),
+            "Only EXPLAIN (LOGICAL | PHYSICAL, JSON FORMAT) is supported.".to_string(),
+        )
+        .into());
+    }
+    if options.explain_type == ExplainType::DistSql && options.explain_format == ExplainFormat::Json
+    {
+        return Err(ErrorCode::NotSupported(
+            "EXPLAIN (TRACE, JSON FORMAT)".to_string(),
+            "Only EXPLAIN (LOGICAL | PHYSICAL, JSON FORMAT) is supported.".to_string(),
+        )
+        .into());
+    }
 
     let mut blocks = Vec::new();
     let result = do_handle_explain(handler_args, options.clone(), stmt, &mut blocks).await;
diff --git a/src/frontend/src/handler/flush.rs b/src/frontend/src/handler/flush.rs
index 58e5a305e8ad4..666dca64583ba 100644
--- a/src/frontend/src/handler/flush.rs
+++ b/src/frontend/src/handler/flush.rs
@@ -26,7 +26,13 @@ pub(super) async fn handle_flush(handler_args: HandlerArgs) -> Result<RwPgRespon
 
 pub(crate) async fn do_flush(session: &SessionImpl) -> Result<()> {
     let client = session.env().meta_client();
-    let version_id = client.flush(true).await?;
+    let database_id = session
+        .env()
+        .catalog_reader()
+        .read_guard()
+        .get_database_by_name(session.database())?
+        .id();
+    let version_id = client.flush(database_id).await?;
 
     // Wait for the snapshot to be synchronized, so that future reads in this session can see
     // previous writes.
diff --git a/src/frontend/src/handler/handle_privilege.rs b/src/frontend/src/handler/handle_privilege.rs
index 851797c7265b3..c70b638c13e2b 100644
--- a/src/frontend/src/handler/handle_privilege.rs
+++ b/src/frontend/src/handler/handle_privilege.rs
@@ -136,6 +136,20 @@ fn make_prost_privilege(
                 grant_objs.push(PbObject::SourceId(source.id));
             }
         }
+        GrantObjects::Sinks(sinks) => {
+            let db_name = session.database();
+            let search_path = session.config().search_path();
+            let user_name = &session.auth_context().user_name;
+
+            for name in sinks {
+                let (schema_name, sink_name) =
+                    Binder::resolve_schema_qualified_name(db_name, name)?;
+                let schema_path = SchemaPath::new(schema_name.as_deref(), &search_path, user_name);
+
+                let (sink, _) = reader.get_sink_by_name(db_name, schema_path, &sink_name)?;
+                grant_objs.push(PbObject::SinkId(sink.id.sink_id));
+            }
+        }
         GrantObjects::AllSourcesInSchema { schemas } => {
             for schema in schemas {
                 let schema_name = Binder::resolve_schema_name(schema)?;
diff --git a/src/frontend/src/handler/mod.rs b/src/frontend/src/handler/mod.rs
index 42de3a116d0b6..9cf94a37c65b0 100644
--- a/src/frontend/src/handler/mod.rs
+++ b/src/frontend/src/handler/mod.rs
@@ -46,6 +46,7 @@ mod alter_set_schema;
 mod alter_source_column;
 mod alter_source_with_sr;
 mod alter_streaming_rate_limit;
+mod alter_swap_rename;
 mod alter_system;
 mod alter_table_column;
 mod alter_table_with_sr;
@@ -336,7 +337,7 @@ pub async fn handle(
             or_replace,
             temporary,
             if_not_exists,
-            source_schema,
+            format_encode,
             source_watermarks,
             append_only,
             on_conflict,
@@ -363,7 +364,7 @@ pub async fn handle(
                 )
                 .await;
             }
-            let source_schema = source_schema.map(|s| s.into_v2_with_warning());
+            let format_encode = format_encode.map(|s| s.into_v2_with_warning());
             create_table::handle_create_table(
                 handler_args,
                 name,
@@ -371,7 +372,7 @@ pub async fn handle(
                 wildcard_idx,
                 constraints,
                 if_not_exists,
-                source_schema,
+                format_encode,
                 source_watermarks,
                 append_only,
                 on_conflict,
@@ -384,19 +385,18 @@ pub async fn handle(
         Statement::CreateDatabase {
             db_name,
             if_not_exists,
-        } => create_database::handle_create_database(handler_args, db_name, if_not_exists).await,
+            owner,
+        } => {
+            create_database::handle_create_database(handler_args, db_name, if_not_exists, owner)
+                .await
+        }
         Statement::CreateSchema {
             schema_name,
             if_not_exists,
-            user_specified,
+            owner,
         } => {
-            create_schema::handle_create_schema(
-                handler_args,
-                schema_name,
-                if_not_exists,
-                user_specified,
-            )
-            .await
+            create_schema::handle_create_schema(handler_args, schema_name, if_not_exists, owner)
+                .await
         }
         Statement::CreateUser(stmt) => create_user::handle_create_user(handler_args, stmt).await,
         Statement::DeclareCursor { stmt } => {
@@ -638,6 +638,18 @@ pub async fn handle(
             )
             .await
         }
+        Statement::AlterSchema {
+            name,
+            operation: AlterSchemaOperation::SwapRenameSchema { target_schema },
+        } => {
+            alter_swap_rename::handle_swap_rename(
+                handler_args,
+                name,
+                target_schema,
+                StatementType::ALTER_SCHEMA,
+            )
+            .await
+        }
         Statement::AlterTable {
             name,
             operation:
@@ -721,6 +733,18 @@ pub async fn handle(
             )
             .await
         }
+        Statement::AlterTable {
+            name,
+            operation: AlterTableOperation::SwapRenameTable { target_table },
+        } => {
+            alter_swap_rename::handle_swap_rename(
+                handler_args,
+                name,
+                target_table,
+                StatementType::ALTER_TABLE,
+            )
+            .await
+        }
         Statement::AlterIndex {
             name,
             operation: AlterIndexOperation::RenameIndex { index_name },
@@ -838,6 +862,19 @@ pub async fn handle(
             )
             .await
         }
+        Statement::AlterView {
+            materialized,
+            name,
+            operation: AlterViewOperation::SwapRenameView { target_view },
+        } => {
+            let statement_type = if materialized {
+                StatementType::ALTER_MATERIALIZED_VIEW
+            } else {
+                StatementType::ALTER_VIEW
+            };
+            alter_swap_rename::handle_swap_rename(handler_args, name, target_view, statement_type)
+                .await
+        }
         Statement::AlterSink {
             name,
             operation: AlterSinkOperation::RenameSink { sink_name },
@@ -885,6 +922,18 @@ pub async fn handle(
             )
             .await
         }
+        Statement::AlterSink {
+            name,
+            operation: AlterSinkOperation::SwapRenameSink { target_sink },
+        } => {
+            alter_swap_rename::handle_swap_rename(
+                handler_args,
+                name,
+                target_sink,
+                StatementType::ALTER_SINK,
+            )
+            .await
+        }
         Statement::AlterSubscription {
             name,
             operation: AlterSubscriptionOperation::RenameSubscription { subscription_name },
@@ -914,6 +963,21 @@ pub async fn handle(
             )
             .await
         }
+        Statement::AlterSubscription {
+            name,
+            operation:
+                AlterSubscriptionOperation::SwapRenameSubscription {
+                    target_subscription,
+                },
+        } => {
+            alter_swap_rename::handle_swap_rename(
+                handler_args,
+                name,
+                target_subscription,
+                StatementType::ALTER_SUBSCRIPTION,
+            )
+            .await
+        }
         Statement::AlterSource {
             name,
             operation: AlterSourceOperation::RenameSource { source_name },
@@ -949,9 +1013,9 @@ pub async fn handle(
         }
         Statement::AlterSource {
             name,
-            operation: AlterSourceOperation::FormatEncode { connector_schema },
+            operation: AlterSourceOperation::FormatEncode { format_encode },
         } => {
-            alter_source_with_sr::handle_alter_source_with_sr(handler_args, name, connector_schema)
+            alter_source_with_sr::handle_alter_source_with_sr(handler_args, name, format_encode)
                 .await
         }
         Statement::AlterSource {
@@ -970,6 +1034,18 @@ pub async fn handle(
             )
             .await
         }
+        Statement::AlterSource {
+            name,
+            operation: AlterSourceOperation::SwapRenameSource { target_source },
+        } => {
+            alter_swap_rename::handle_swap_rename(
+                handler_args,
+                name,
+                target_source,
+                StatementType::ALTER_SOURCE,
+            )
+            .await
+        }
         Statement::AlterFunction {
             name,
             args,
diff --git a/src/frontend/src/handler/show.rs b/src/frontend/src/handler/show.rs
index 0da3e76ce3fe6..d8ec8b44d827e 100644
--- a/src/frontend/src/handler/show.rs
+++ b/src/frontend/src/handler/show.rs
@@ -633,7 +633,7 @@ pub fn handle_show_create_object(
             let view = schema
                 .get_view_by_name(&object_name)
                 .ok_or_else(|| CatalogError::NotFound("view", name.to_string()))?;
-            view.create_sql()
+            view.create_sql(schema.name())
         }
         ShowCreateType::Table => {
             let table = schema
diff --git a/src/frontend/src/handler/util.rs b/src/frontend/src/handler/util.rs
index 5551154bd8ec4..169716cd504a2 100644
--- a/src/frontend/src/handler/util.rs
+++ b/src/frontend/src/handler/util.rs
@@ -35,8 +35,8 @@ use risingwave_common::types::{
 use risingwave_common::util::epoch::Epoch;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_sqlparser::ast::{
-    CompatibleSourceSchema, ConnectorSchema, ObjectName, Query, Select, SelectItem, SetExpr,
-    TableFactor, TableWithJoins,
+    CompatibleFormatEncode, Expr, FormatEncodeOptions, Ident, ObjectName, OrderByExpr, Query,
+    Select, SelectItem, SetExpr, TableFactor, TableWithJoins,
 };
 use thiserror_ext::AsReport;
 
@@ -194,16 +194,16 @@ pub fn to_pg_field(f: &Field) -> PgFieldDescriptor {
 }
 
 #[easy_ext::ext(SourceSchemaCompatExt)]
-impl CompatibleSourceSchema {
-    /// Convert `self` to [`ConnectorSchema`] and warn the user if the syntax is deprecated.
-    pub fn into_v2_with_warning(self) -> ConnectorSchema {
+impl CompatibleFormatEncode {
+    /// Convert `self` to [`FormatEncodeOptions`] and warn the user if the syntax is deprecated.
+    pub fn into_v2_with_warning(self) -> FormatEncodeOptions {
         match self {
-            CompatibleSourceSchema::RowFormat(inner) => {
+            CompatibleFormatEncode::RowFormat(inner) => {
                 // TODO: should be warning
                 current::notice_to_user("RisingWave will stop supporting the syntax \"ROW FORMAT\" in future versions, which will be changed to \"FORMAT ... ENCODE ...\" syntax.");
-                inner.into_source_schema_v2()
+                inner.into_format_encode_v2()
             }
-            CompatibleSourceSchema::V2(inner) => inner,
+            CompatibleFormatEncode::V2(inner) => inner,
         }
     }
 }
@@ -234,6 +234,22 @@ pub fn gen_query_from_table_name(from_name: ObjectName) -> Query {
     }
 }
 
+pub fn gen_query_from_table_name_order_by(from_name: ObjectName, pk_names: Vec<String>) -> Query {
+    let mut query = gen_query_from_table_name(from_name);
+    query.order_by = pk_names
+        .into_iter()
+        .map(|pk| {
+            let expr = Expr::Identifier(Ident::with_quote_unchecked('"', pk));
+            OrderByExpr {
+                expr,
+                asc: None,
+                nulls_first: None,
+            }
+        })
+        .collect();
+    query
+}
+
 pub fn convert_unix_millis_to_logstore_u64(unix_millis: u64) -> u64 {
     Epoch::from_unix_millis(unix_millis).0
 }
diff --git a/src/frontend/src/handler/variable.rs b/src/frontend/src/handler/variable.rs
index 96fd232215ccd..e22ab045565d3 100644
--- a/src/frontend/src/handler/variable.rs
+++ b/src/frontend/src/handler/variable.rs
@@ -54,7 +54,7 @@ pub fn handle_set(
         status: &'a mut ParameterStatus,
     }
 
-    impl<'a> ConfigReporter for Reporter<'a> {
+    impl ConfigReporter for Reporter<'_> {
         fn report_status(&mut self, key: &str, new_val: String) {
             if key == "APPLICATION_NAME" {
                 self.status.application_name = Some(new_val);
diff --git a/src/frontend/src/lib.rs b/src/frontend/src/lib.rs
index 8dbd09a19a629..5c006e191157e 100644
--- a/src/frontend/src/lib.rs
+++ b/src/frontend/src/lib.rs
@@ -33,7 +33,6 @@
 #![feature(error_generic_member_access)]
 #![feature(iterator_try_collect)]
 #![feature(used_with_arg)]
-#![feature(entry_insert)]
 #![recursion_limit = "256"]
 
 #[cfg(test)]
diff --git a/src/frontend/src/meta_client.rs b/src/frontend/src/meta_client.rs
index bdbc67a4b9720..a91a0d8abc878 100644
--- a/src/frontend/src/meta_client.rs
+++ b/src/frontend/src/meta_client.rs
@@ -39,6 +39,8 @@ use risingwave_pb::meta::{EventLog, PbThrottleTarget, RecoveryStatus};
 use risingwave_rpc_client::error::Result;
 use risingwave_rpc_client::{HummockMetaClient, MetaClient};
 
+use crate::catalog::DatabaseId;
+
 /// A wrapper around the `MetaClient` that only provides a minor set of meta rpc.
 /// Most of the rpc to meta are delegated by other separate structs like `CatalogWriter`,
 /// `WorkerNodeManager`, etc. So frontend rarely needs to call `MetaClient` directly.
@@ -48,7 +50,7 @@ use risingwave_rpc_client::{HummockMetaClient, MetaClient};
 pub trait FrontendMetaClient: Send + Sync {
     async fn try_unregister(&self);
 
-    async fn flush(&self, checkpoint: bool) -> Result<HummockVersionId>;
+    async fn flush(&self, database_id: DatabaseId) -> Result<HummockVersionId>;
 
     async fn wait(&self) -> Result<()>;
 
@@ -133,8 +135,8 @@ impl FrontendMetaClient for FrontendMetaClientImpl {
         self.0.try_unregister().await;
     }
 
-    async fn flush(&self, checkpoint: bool) -> Result<HummockVersionId> {
-        self.0.flush(checkpoint).await
+    async fn flush(&self, database_id: DatabaseId) -> Result<HummockVersionId> {
+        self.0.flush(database_id).await
     }
 
     async fn wait(&self) -> Result<()> {
diff --git a/src/frontend/src/observer/observer_manager.rs b/src/frontend/src/observer/observer_manager.rs
index b1ecf4182d1d0..92408c2b03885 100644
--- a/src/frontend/src/observer/observer_manager.rs
+++ b/src/frontend/src/observer/observer_manager.rs
@@ -245,6 +245,7 @@ impl FrontendObserverNode {
         let Some(info) = resp.info.as_ref() else {
             return;
         };
+        tracing::trace!(op = ?resp.operation(), ?info, "handle catalog notification");
 
         let mut catalog_guard = self.catalog.write();
         match info {
diff --git a/src/frontend/src/optimizer/logical_optimization.rs b/src/frontend/src/optimizer/logical_optimization.rs
index a03bb95a0d51e..e413188154569 100644
--- a/src/frontend/src/optimizer/logical_optimization.rs
+++ b/src/frontend/src/optimizer/logical_optimization.rs
@@ -108,6 +108,8 @@ impl OptimizationStage {
 
 use std::sync::LazyLock;
 
+use risingwave_sqlparser::ast::ExplainFormat;
+
 pub struct LogicalOptimizer {}
 
 static DAG_TO_TREE: LazyLock<OptimizationStage> = LazyLock::new(|| {
@@ -134,6 +136,8 @@ static TABLE_FUNCTION_CONVERT: LazyLock<OptimizationStage> = LazyLock::new(|| {
             TableFunctionToFileScanRule::create(),
             // Apply postgres query rule next
             TableFunctionToPostgresQueryRule::create(),
+            // Apply mysql query rule next
+            TableFunctionToMySqlQueryRule::create(),
             // Apply project set rule last
             TableFunctionToProjectSetRule::create(),
         ],
@@ -157,6 +161,14 @@ static TABLE_FUNCTION_TO_POSTGRES_QUERY: LazyLock<OptimizationStage> = LazyLock:
     )
 });
 
+static TABLE_FUNCTION_TO_MYSQL_QUERY: LazyLock<OptimizationStage> = LazyLock::new(|| {
+    OptimizationStage::new(
+        "Table Function To MySQL",
+        vec![TableFunctionToMySqlQueryRule::create()],
+        ApplyOrder::TopDown,
+    )
+});
+
 static VALUES_EXTRACT_PROJECT: LazyLock<OptimizationStage> = LazyLock::new(|| {
     OptimizationStage::new(
         "Values Extract Project",
@@ -197,12 +209,7 @@ static GENERAL_UNNESTING_TRANS_APPLY_WITH_SHARE: LazyLock<OptimizationStage> =
     LazyLock::new(|| {
         OptimizationStage::new(
             "General Unnesting(Translate Apply)",
-            vec![
-                TranslateApplyRule::create(true),
-                // Separate the project from a join if necessary because `ApplyJoinTransposeRule`
-                // can't handle a join with `output_indices`.
-                ProjectJoinSeparateRule::create(),
-            ],
+            vec![TranslateApplyRule::create(true)],
             ApplyOrder::TopDown,
         )
     });
@@ -211,12 +218,7 @@ static GENERAL_UNNESTING_TRANS_APPLY_WITHOUT_SHARE: LazyLock<OptimizationStage>
     LazyLock::new(|| {
         OptimizationStage::new(
             "General Unnesting(Translate Apply)",
-            vec![
-                TranslateApplyRule::create(false),
-                // Separate the project from a join if necessary because `ApplyJoinTransposeRule`
-                // can't handle a join with `output_indices`.
-                ProjectJoinSeparateRule::create(),
-            ],
+            vec![TranslateApplyRule::create(false)],
             ApplyOrder::TopDown,
         )
     });
@@ -684,7 +686,14 @@ impl LogicalOptimizer {
         InputRefValidator.validate(plan.clone());
 
         if ctx.is_explain_logical() {
-            ctx.store_logical(plan.explain_to_string());
+            match ctx.explain_format() {
+                ExplainFormat::Text => {
+                    ctx.store_logical(plan.explain_to_string());
+                }
+                ExplainFormat::Json => {
+                    ctx.store_logical(plan.explain_to_json());
+                }
+            }
         }
 
         Ok(plan)
@@ -714,6 +723,7 @@ impl LogicalOptimizer {
         // Table function should be converted into `file_scan` before `project_set`.
         plan = plan.optimize_by_rules(&TABLE_FUNCTION_TO_FILE_SCAN);
         plan = plan.optimize_by_rules(&TABLE_FUNCTION_TO_POSTGRES_QUERY);
+        plan = plan.optimize_by_rules(&TABLE_FUNCTION_TO_MYSQL_QUERY);
         // In order to unnest a table function, we need to convert it into a `project_set` first.
         plan = plan.optimize_by_rules(&TABLE_FUNCTION_CONVERT);
 
@@ -789,7 +799,14 @@ impl LogicalOptimizer {
         InputRefValidator.validate(plan.clone());
 
         if ctx.is_explain_logical() {
-            ctx.store_logical(plan.explain_to_string());
+            match ctx.explain_format() {
+                ExplainFormat::Text => {
+                    ctx.store_logical(plan.explain_to_string());
+                }
+                ExplainFormat::Json => {
+                    ctx.store_logical(plan.explain_to_json());
+                }
+            }
         }
 
         Ok(plan)
diff --git a/src/frontend/src/optimizer/optimizer_context.rs b/src/frontend/src/optimizer/optimizer_context.rs
index 35c5a3ee9129b..75af6a3b3da69 100644
--- a/src/frontend/src/optimizer/optimizer_context.rs
+++ b/src/frontend/src/optimizer/optimizer_context.rs
@@ -19,7 +19,7 @@ use std::marker::PhantomData;
 use std::rc::Rc;
 use std::sync::Arc;
 
-use risingwave_sqlparser::ast::{ExplainOptions, ExplainType};
+use risingwave_sqlparser::ast::{ExplainFormat, ExplainOptions, ExplainType};
 
 use crate::binder::ShareId;
 use crate::expr::{CorrelatedId, SessionTimezone};
@@ -176,6 +176,10 @@ impl OptimizerContext {
         self.explain_options.explain_type.clone()
     }
 
+    pub fn explain_format(&self) -> ExplainFormat {
+        self.explain_options.explain_format.clone()
+    }
+
     pub fn is_explain_logical(&self) -> bool {
         self.explain_type() == ExplainType::Logical
     }
diff --git a/src/frontend/src/optimizer/plan_node/batch_log_seq_scan.rs b/src/frontend/src/optimizer/plan_node/batch_log_seq_scan.rs
index de8cadda4502d..47dd68ceb52ef 100644
--- a/src/frontend/src/optimizer/plan_node/batch_log_seq_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/batch_log_seq_scan.rs
@@ -35,7 +35,7 @@ pub struct BatchLogSeqScan {
 
 impl BatchLogSeqScan {
     fn new_inner(core: generic::LogScan, dist: Distribution) -> Self {
-        let order = Order::any();
+        let order = Order::new(core.table_desc.pk.clone());
         let base = PlanBase::new_batch(core.ctx(), core.schema(), dist, order);
 
         Self { base, core }
@@ -88,8 +88,11 @@ impl Distill for BatchLogSeqScan {
             });
             vec.push(("distribution", dist));
         }
+        vec.push(("old_epoch", Pretty::from(self.core.old_epoch.to_string())));
+        vec.push(("new_epoch", Pretty::from(self.core.new_epoch.to_string())));
+        vec.push(("version_id", Pretty::from(self.core.version_id.to_string())));
 
-        childless_record("BatchScan", vec)
+        childless_record("BatchLogSeqScan", vec)
     }
 }
 
@@ -126,6 +129,8 @@ impl TryToBatchPb for BatchLogSeqScan {
                     },
                 )),
             }),
+            // It's currently true.
+            ordered: !self.order().is_any(),
         }))
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/batch_mysql_query.rs b/src/frontend/src/optimizer/plan_node/batch_mysql_query.rs
new file mode 100644
index 0000000000000..308b1e82c63f3
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/batch_mysql_query.rs
@@ -0,0 +1,96 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pretty_xmlish::XmlNode;
+use risingwave_pb::batch_plan::plan_node::NodeBody;
+use risingwave_pb::batch_plan::MySqlQueryNode;
+
+use super::batch::prelude::*;
+use super::utils::{childless_record, column_names_pretty, Distill};
+use super::{
+    generic, ExprRewritable, PlanBase, PlanRef, ToBatchPb, ToDistributedBatch, ToLocalBatch,
+};
+use crate::error::Result;
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::property::{Distribution, Order};
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct BatchMySqlQuery {
+    pub base: PlanBase<Batch>,
+    pub core: generic::MySqlQuery,
+}
+
+impl BatchMySqlQuery {
+    pub fn new(core: generic::MySqlQuery) -> Self {
+        let base = PlanBase::new_batch_with_core(&core, Distribution::Single, Order::any());
+
+        Self { base, core }
+    }
+
+    pub fn column_names(&self) -> Vec<&str> {
+        self.schema().names_str()
+    }
+
+    pub fn clone_with_dist(&self) -> Self {
+        let base = self.base.clone_with_new_distribution(Distribution::Single);
+        Self {
+            base,
+            core: self.core.clone(),
+        }
+    }
+}
+
+impl_plan_tree_node_for_leaf! { BatchMySqlQuery }
+
+impl Distill for BatchMySqlQuery {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let fields = vec![("columns", column_names_pretty(self.schema()))];
+        childless_record("BatchMySqlQuery", fields)
+    }
+}
+
+impl ToLocalBatch for BatchMySqlQuery {
+    fn to_local(&self) -> Result<PlanRef> {
+        Ok(self.clone_with_dist().into())
+    }
+}
+
+impl ToDistributedBatch for BatchMySqlQuery {
+    fn to_distributed(&self) -> Result<PlanRef> {
+        Ok(self.clone_with_dist().into())
+    }
+}
+
+impl ToBatchPb for BatchMySqlQuery {
+    fn to_batch_prost_body(&self) -> NodeBody {
+        NodeBody::MysqlQuery(MySqlQueryNode {
+            columns: self
+                .core
+                .columns()
+                .iter()
+                .map(|c| c.to_protobuf())
+                .collect(),
+            hostname: self.core.hostname.clone(),
+            port: self.core.port.clone(),
+            username: self.core.username.clone(),
+            password: self.core.password.clone(),
+            database: self.core.database.clone(),
+            query: self.core.query.clone(),
+        })
+    }
+}
+
+impl ExprRewritable for BatchMySqlQuery {}
+
+impl ExprVisitable for BatchMySqlQuery {}
diff --git a/src/frontend/src/optimizer/plan_node/generic/join.rs b/src/frontend/src/optimizer/plan_node/generic/join.rs
index f7ce096e73eb2..4663205ea6f09 100644
--- a/src/frontend/src/optimizer/plan_node/generic/join.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/join.rs
@@ -492,7 +492,6 @@ impl<PlanRef: GenericPlanRef> Join<PlanRef> {
 /// predicate.
 ///
 /// `InputRef`s in the right pushed condition are indexed by the right child's output schema.
-
 pub fn push_down_into_join(
     predicate: &mut Condition,
     left_col_num: usize,
@@ -534,7 +533,6 @@ pub fn push_down_into_join(
 /// pushed part will be removed from the original join predicate.
 ///
 /// `InputRef`s in the right pushed condition are indexed by the right child's output schema.
-
 pub fn push_down_join_condition(
     on_condition: &mut Condition,
     left_col_num: usize,
diff --git a/src/frontend/src/optimizer/plan_node/generic/log_scan.rs b/src/frontend/src/optimizer/plan_node/generic/log_scan.rs
index 793cbaea4f205..a57ba79242d1a 100644
--- a/src/frontend/src/optimizer/plan_node/generic/log_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/log_scan.rs
@@ -16,6 +16,8 @@ use std::collections::HashMap;
 use std::rc::Rc;
 
 use educe::Educe;
+use fixedbitset::FixedBitSet;
+use itertools::Itertools;
 use pretty_xmlish::Pretty;
 use risingwave_common::catalog::{Field, Schema, TableDesc};
 use risingwave_common::types::DataType;
@@ -32,7 +34,9 @@ const OP_TYPE: DataType = DataType::Varchar;
 #[educe(PartialEq, Eq, Hash)]
 pub struct LogScan {
     pub table_name: String,
-    /// Include `output_col_idx` and `op_column`
+    /// Include `output_col_idx_with_out_hidden` and `op_column`
+    pub output_col_idx_with_out_hidden: Vec<usize>,
+    /// Include `output_col_idx_with_out_hidden` and `op_column` and hidden pk
     pub output_col_idx: Vec<usize>,
     /// Descriptor of the table
     pub table_desc: Rc<TableDesc>,
@@ -81,6 +85,16 @@ impl LogScan {
         out_column_names
     }
 
+    pub(crate) fn column_names_without_hidden(&self) -> Vec<String> {
+        let mut out_column_names: Vec<_> = self
+            .output_col_idx_with_out_hidden
+            .iter()
+            .map(|&i| self.table_desc.columns[i].name.clone())
+            .collect();
+        out_column_names.push(OP_NAME.to_string());
+        out_column_names
+    }
+
     pub fn distribution_key(&self) -> Option<Vec<usize>> {
         let tb_idx_to_op_idx = self
             .output_col_idx
@@ -98,6 +112,7 @@ impl LogScan {
     /// Create a logical scan node for log table scan
     pub(crate) fn new(
         table_name: String,
+        output_col_idx_with_out_hidden: Vec<usize>,
         output_col_idx: Vec<usize>,
         table_desc: Rc<TableDesc>,
         ctx: OptimizerContextRef,
@@ -107,6 +122,7 @@ impl LogScan {
     ) -> Self {
         Self {
             table_name,
+            output_col_idx_with_out_hidden,
             output_col_idx,
             table_desc,
             chunk_size: None,
@@ -145,17 +161,22 @@ impl LogScan {
         Schema { fields }
     }
 
-    pub(crate) fn schema_without_table_name(&self) -> Schema {
-        let mut fields: Vec<_> = self
+    pub(crate) fn out_fields(&self) -> FixedBitSet {
+        let mut out_fields_vec = self
             .output_col_idx
             .iter()
-            .map(|tb_idx| {
-                let col = &self.table_desc.columns[*tb_idx];
-                Field::from(col)
+            .enumerate()
+            .filter_map(|(index, idx)| {
+                if self.output_col_idx_with_out_hidden.contains(idx) {
+                    Some(index)
+                } else {
+                    None
+                }
             })
-            .collect();
-        fields.push(Field::with_name(OP_TYPE, OP_NAME));
-        Schema { fields }
+            .collect_vec();
+        // add op column
+        out_fields_vec.push(self.output_col_idx.len());
+        FixedBitSet::from_iter(out_fields_vec)
     }
 
     pub(crate) fn ctx(&self) -> OptimizerContextRef {
diff --git a/src/frontend/src/optimizer/plan_node/generic/mod.rs b/src/frontend/src/optimizer/plan_node/generic/mod.rs
index 6a076025b906c..c35a367e8ccec 100644
--- a/src/frontend/src/optimizer/plan_node/generic/mod.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/mod.rs
@@ -92,6 +92,9 @@ pub use file_scan::*;
 mod postgres_query;
 pub use postgres_query::*;
 
+mod mysql_query;
+pub use mysql_query::*;
+
 pub trait DistillUnit {
     fn distill_with_name<'a>(&self, name: impl Into<Cow<'a, str>>) -> XmlNode<'a>;
 }
diff --git a/src/frontend/src/optimizer/plan_node/generic/mysql_query.rs b/src/frontend/src/optimizer/plan_node/generic/mysql_query.rs
new file mode 100644
index 0000000000000..03bbfa0b229eb
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/generic/mysql_query.rs
@@ -0,0 +1,67 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use educe::Educe;
+use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema};
+
+use super::GenericPlanNode;
+use crate::optimizer::optimizer_context::OptimizerContextRef;
+use crate::optimizer::property::FunctionalDependencySet;
+
+#[derive(Debug, Clone, Educe)]
+#[educe(PartialEq, Eq, Hash)]
+pub struct MySqlQuery {
+    pub schema: Schema,
+    pub hostname: String,
+    pub port: String,
+    pub username: String,
+    pub password: String,
+    pub database: String,
+    pub query: String,
+
+    #[educe(PartialEq(ignore))]
+    #[educe(Hash(ignore))]
+    pub ctx: OptimizerContextRef,
+}
+
+impl GenericPlanNode for MySqlQuery {
+    fn schema(&self) -> Schema {
+        self.schema.clone()
+    }
+
+    fn stream_key(&self) -> Option<Vec<usize>> {
+        None
+    }
+
+    fn ctx(&self) -> OptimizerContextRef {
+        self.ctx.clone()
+    }
+
+    fn functional_dependency(&self) -> FunctionalDependencySet {
+        FunctionalDependencySet::new(self.schema.len())
+    }
+}
+
+impl MySqlQuery {
+    pub fn columns(&self) -> Vec<ColumnDesc> {
+        self.schema
+            .fields
+            .iter()
+            .enumerate()
+            .map(|(i, f)| {
+                ColumnDesc::named(f.name.clone(), ColumnId::new(i as i32), f.data_type.clone())
+            })
+            .collect()
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_node/generic/over_window.rs b/src/frontend/src/optimizer/plan_node/generic/over_window.rs
index fc10df60421bb..c65dabd3ee09c 100644
--- a/src/frontend/src/optimizer/plan_node/generic/over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/generic/over_window.rs
@@ -45,7 +45,7 @@ struct PlanWindowFunctionDisplay<'a> {
     pub input_schema: &'a Schema,
 }
 
-impl<'a> std::fmt::Debug for PlanWindowFunctionDisplay<'a> {
+impl std::fmt::Debug for PlanWindowFunctionDisplay<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         let window_function = self.window_function;
         if f.alternate() {
diff --git a/src/frontend/src/optimizer/plan_node/logical_agg.rs b/src/frontend/src/optimizer/plan_node/logical_agg.rs
index 4e2474287c969..7f2b527979242 100644
--- a/src/frontend/src/optimizer/plan_node/logical_agg.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_agg.rs
@@ -440,7 +440,6 @@ impl LogicalAgg {
     ///  `MergeProject`       z
     ///  /        \
     /// x          y
-
     fn build_approx_percentile_aggs(
         &self,
         input: PlanRef,
diff --git a/src/frontend/src/optimizer/plan_node/logical_join.rs b/src/frontend/src/optimizer/plan_node/logical_join.rs
index d7e8eb79dd70d..98469f4d05421 100644
--- a/src/frontend/src/optimizer/plan_node/logical_join.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_join.rs
@@ -997,16 +997,7 @@ impl LogicalJoin {
         result_plan
     }
 
-    fn to_stream_temporal_join(
-        &self,
-        predicate: EqJoinPredicate,
-        ctx: &mut ToStreamContext,
-    ) -> Result<StreamTemporalJoin> {
-        use super::stream::prelude::*;
-
-        assert!(predicate.has_eq());
-
-        let right = self.right();
+    fn check_temporal_rhs(right: &PlanRef) -> Result<&LogicalScan> {
         let Some(logical_scan) = right.as_logical_scan() else {
             return Err(RwError::from(ErrorCode::NotSupported(
                 "Temporal join requires a table scan as its lookup table".into(),
@@ -1020,6 +1011,88 @@ impl LogicalJoin {
                 "Please use FOR SYSTEM_TIME AS OF PROCTIME() syntax".into(),
             )));
         }
+        Ok(logical_scan)
+    }
+
+    fn temporal_join_scan_predicate_pull_up(
+        logical_scan: &LogicalScan,
+        predicate: EqJoinPredicate,
+        output_indices: &[usize],
+        left_schema_len: usize,
+    ) -> Result<(StreamTableScan, EqJoinPredicate, Condition, Vec<usize>)> {
+        // Extract the predicate from logical scan. Only pure scan is supported.
+        let (new_scan, scan_predicate, project_expr) = logical_scan.predicate_pull_up();
+        // Construct output column to require column mapping
+        let o2r = if let Some(project_expr) = project_expr {
+            project_expr
+                .into_iter()
+                .map(|x| x.as_input_ref().unwrap().index)
+                .collect_vec()
+        } else {
+            (0..logical_scan.output_col_idx().len()).collect_vec()
+        };
+        let mut join_predicate_rewriter = LookupJoinPredicateRewriter {
+            offset: left_schema_len,
+            mapping: o2r.clone(),
+        };
+
+        let new_eq_cond = predicate
+            .eq_cond()
+            .rewrite_expr(&mut join_predicate_rewriter);
+
+        let mut scan_predicate_rewriter = LookupJoinScanPredicateRewriter {
+            offset: left_schema_len,
+        };
+
+        let new_other_cond = predicate
+            .other_cond()
+            .clone()
+            .rewrite_expr(&mut join_predicate_rewriter)
+            .and(scan_predicate.rewrite_expr(&mut scan_predicate_rewriter));
+
+        let new_join_on = new_eq_cond.and(new_other_cond);
+
+        let new_predicate = EqJoinPredicate::create(
+            left_schema_len,
+            new_scan.schema().len(),
+            new_join_on.clone(),
+        );
+
+        // Rewrite the join output indices and all output indices referred to the old scan need to
+        // rewrite.
+        let new_join_output_indices = output_indices
+            .iter()
+            .map(|&x| {
+                if x < left_schema_len {
+                    x
+                } else {
+                    o2r[x - left_schema_len] + left_schema_len
+                }
+            })
+            .collect_vec();
+        // Use UpstreamOnly chain type
+        let new_stream_table_scan =
+            StreamTableScan::new_with_stream_scan_type(new_scan, StreamScanType::UpstreamOnly);
+        Ok((
+            new_stream_table_scan,
+            new_predicate,
+            new_join_on,
+            new_join_output_indices,
+        ))
+    }
+
+    fn to_stream_temporal_join(
+        &self,
+        predicate: EqJoinPredicate,
+        ctx: &mut ToStreamContext,
+    ) -> Result<StreamTemporalJoin> {
+        use super::stream::prelude::*;
+
+        assert!(predicate.has_eq());
+
+        let right = self.right();
+
+        let logical_scan = Self::check_temporal_rhs(&right)?;
 
         let table_desc = logical_scan.table_desc();
         let output_column_ids = logical_scan.output_column_ids();
@@ -1085,67 +1158,79 @@ impl LogicalJoin {
         // Enforce a shuffle for the temporal join LHS to let the scheduler be able to schedule the join fragment together with the RHS with a `no_shuffle` exchange.
         let left = required_dist.enforce(left, &Order::any());
 
-        // Extract the predicate from logical scan. Only pure scan is supported.
-        let (new_scan, scan_predicate, project_expr) = logical_scan.predicate_pull_up();
-        // Construct output column to require column mapping
-        let o2r = if let Some(project_expr) = project_expr {
-            project_expr
-                .into_iter()
-                .map(|x| x.as_input_ref().unwrap().index)
-                .collect_vec()
-        } else {
-            (0..logical_scan.output_col_idx().len()).collect_vec()
-        };
-        let left_schema_len = self.left().schema().len();
-        let mut join_predicate_rewriter = LookupJoinPredicateRewriter {
-            offset: left_schema_len,
-            mapping: o2r.clone(),
-        };
+        let (new_stream_table_scan, new_predicate, new_join_on, new_join_output_indices) =
+            Self::temporal_join_scan_predicate_pull_up(
+                logical_scan,
+                predicate,
+                self.output_indices(),
+                self.left().schema().len(),
+            )?;
 
-        let new_eq_cond = predicate
-            .eq_cond()
-            .rewrite_expr(&mut join_predicate_rewriter);
+        let right = RequiredDist::no_shuffle(new_stream_table_scan.into());
+        if !new_predicate.has_eq() {
+            return Err(RwError::from(ErrorCode::NotSupported(
+                "Temporal join requires a non trivial join condition".into(),
+                "Please remove the false condition of the join".into(),
+            )));
+        }
 
-        let mut scan_predicate_rewriter = LookupJoinScanPredicateRewriter {
-            offset: left_schema_len,
-        };
+        // Construct a new logical join, because we have change its RHS.
+        let new_logical_join = generic::Join::new(
+            left,
+            right,
+            new_join_on,
+            self.join_type(),
+            new_join_output_indices,
+        );
 
-        let new_other_cond = predicate
-            .other_cond()
-            .clone()
-            .rewrite_expr(&mut join_predicate_rewriter)
-            .and(scan_predicate.rewrite_expr(&mut scan_predicate_rewriter));
+        let new_predicate = new_predicate.retain_prefix_eq_key(lookup_prefix_len);
 
-        let new_join_on = new_eq_cond.and(new_other_cond);
-        let new_predicate = EqJoinPredicate::create(
-            left_schema_len,
-            new_scan.schema().len(),
-            new_join_on.clone(),
-        );
+        Ok(StreamTemporalJoin::new(
+            new_logical_join,
+            new_predicate,
+            false,
+        ))
+    }
 
-        if !new_predicate.has_eq() {
+    fn to_stream_nested_loop_temporal_join(
+        &self,
+        predicate: EqJoinPredicate,
+        ctx: &mut ToStreamContext,
+    ) -> Result<StreamTemporalJoin> {
+        use super::stream::prelude::*;
+        assert!(!predicate.has_eq());
+
+        let left = self.left().to_stream_with_dist_required(
+            &RequiredDist::PhysicalDist(Distribution::Broadcast),
+            ctx,
+        )?;
+        assert!(left.as_stream_exchange().is_some());
+
+        if self.join_type() != JoinType::Inner {
             return Err(RwError::from(ErrorCode::NotSupported(
-                "Temporal join requires a non trivial join condition".into(),
-                "Please remove the false condition of the join".into(),
+                "Temporal join requires an inner join".into(),
+                "Please use an inner join".into(),
             )));
         }
 
-        // Rewrite the join output indices and all output indices referred to the old scan need to
-        // rewrite.
-        let new_join_output_indices = self
-            .output_indices()
-            .iter()
-            .map(|&x| {
-                if x < left_schema_len {
-                    x
-                } else {
-                    o2r[x - left_schema_len] + left_schema_len
-                }
-            })
-            .collect_vec();
-        // Use UpstreamOnly chain type
-        let new_stream_table_scan =
-            StreamTableScan::new_with_stream_scan_type(new_scan, StreamScanType::UpstreamOnly);
+        if !left.append_only() {
+            return Err(RwError::from(ErrorCode::NotSupported(
+                "Nested-loop Temporal join requires the left hash side to be append only".into(),
+                "Please ensure the left hash side is append only".into(),
+            )));
+        }
+
+        let right = self.right();
+        let logical_scan = Self::check_temporal_rhs(&right)?;
+
+        let (new_stream_table_scan, new_predicate, new_join_on, new_join_output_indices) =
+            Self::temporal_join_scan_predicate_pull_up(
+                logical_scan,
+                predicate,
+                self.output_indices(),
+                self.left().schema().len(),
+            )?;
+
         let right = RequiredDist::no_shuffle(new_stream_table_scan.into());
 
         // Construct a new logical join, because we have change its RHS.
@@ -1157,9 +1242,11 @@ impl LogicalJoin {
             new_join_output_indices,
         );
 
-        let new_predicate = new_predicate.retain_prefix_eq_key(lookup_prefix_len);
-
-        Ok(StreamTemporalJoin::new(new_logical_join, new_predicate))
+        Ok(StreamTemporalJoin::new(
+            new_logical_join,
+            new_predicate,
+            true,
+        ))
     }
 
     fn to_stream_dynamic_filter(
@@ -1383,6 +1470,9 @@ impl ToStream for LogicalJoin {
             } else {
                 self.to_stream_hash_join(predicate, ctx)
             }
+        } else if self.should_be_temporal_join() {
+            self.to_stream_nested_loop_temporal_join(predicate, ctx)
+                .map(|x| x.into())
         } else if let Some(dynamic_filter) =
             self.to_stream_dynamic_filter(self.on().clone(), ctx)?
         {
diff --git a/src/frontend/src/optimizer/plan_node/logical_mysql_query.rs b/src/frontend/src/optimizer/plan_node/logical_mysql_query.rs
new file mode 100644
index 0000000000000..1512fe60120a3
--- /dev/null
+++ b/src/frontend/src/optimizer/plan_node/logical_mysql_query.rs
@@ -0,0 +1,115 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use pretty_xmlish::XmlNode;
+use risingwave_common::bail;
+use risingwave_common::catalog::Schema;
+
+use super::generic::GenericPlanRef;
+use super::utils::{childless_record, Distill};
+use super::{
+    generic, BatchMySqlQuery, ColPrunable, ExprRewritable, Logical, LogicalProject, PlanBase,
+    PlanRef, PredicatePushdown, ToBatch, ToStream,
+};
+use crate::error::Result;
+use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
+use crate::optimizer::plan_node::utils::column_names_pretty;
+use crate::optimizer::plan_node::{
+    ColumnPruningContext, LogicalFilter, PredicatePushdownContext, RewriteStreamContext,
+    ToStreamContext,
+};
+use crate::utils::{ColIndexMapping, Condition};
+use crate::OptimizerContextRef;
+
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LogicalMySqlQuery {
+    pub base: PlanBase<Logical>,
+    pub core: generic::MySqlQuery,
+}
+
+impl LogicalMySqlQuery {
+    pub fn new(
+        ctx: OptimizerContextRef,
+        schema: Schema,
+        hostname: String,
+        port: String,
+        username: String,
+        password: String,
+        database: String,
+        query: String,
+    ) -> Self {
+        let core = generic::MySqlQuery {
+            schema,
+            hostname,
+            port,
+            username,
+            password,
+            database,
+            query,
+            ctx,
+        };
+
+        let base = PlanBase::new_logical_with_core(&core);
+
+        LogicalMySqlQuery { base, core }
+    }
+}
+
+impl_plan_tree_node_for_leaf! {LogicalMySqlQuery}
+impl Distill for LogicalMySqlQuery {
+    fn distill<'a>(&self) -> XmlNode<'a> {
+        let fields = vec![("columns", column_names_pretty(self.schema()))];
+        childless_record("LogicalMySqlQuery", fields)
+    }
+}
+
+impl ColPrunable for LogicalMySqlQuery {
+    fn prune_col(&self, required_cols: &[usize], _ctx: &mut ColumnPruningContext) -> PlanRef {
+        LogicalProject::with_out_col_idx(self.clone().into(), required_cols.iter().cloned()).into()
+    }
+}
+
+impl ExprRewritable for LogicalMySqlQuery {}
+
+impl ExprVisitable for LogicalMySqlQuery {}
+
+impl PredicatePushdown for LogicalMySqlQuery {
+    fn predicate_pushdown(
+        &self,
+        predicate: Condition,
+        _ctx: &mut PredicatePushdownContext,
+    ) -> PlanRef {
+        // No pushdown.
+        LogicalFilter::create(self.clone().into(), predicate)
+    }
+}
+
+impl ToBatch for LogicalMySqlQuery {
+    fn to_batch(&self) -> Result<PlanRef> {
+        Ok(BatchMySqlQuery::new(self.core.clone()).into())
+    }
+}
+
+impl ToStream for LogicalMySqlQuery {
+    fn to_stream(&self, _ctx: &mut ToStreamContext) -> Result<PlanRef> {
+        bail!("mysql_query function is not supported in streaming mode")
+    }
+
+    fn logical_rewrite_for_stream(
+        &self,
+        _ctx: &mut RewriteStreamContext,
+    ) -> Result<(PlanRef, ColIndexMapping)> {
+        bail!("mysql_query function is not supported in streaming mode")
+    }
+}
diff --git a/src/frontend/src/optimizer/plan_node/logical_over_window.rs b/src/frontend/src/optimizer/plan_node/logical_over_window.rs
index a790e006ddd97..0e4e56ac2a200 100644
--- a/src/frontend/src/optimizer/plan_node/logical_over_window.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_over_window.rs
@@ -151,7 +151,7 @@ impl<'a> LogicalOverWindowBuilder<'a> {
     }
 }
 
-impl<'a> ExprRewriter for LogicalOverWindowBuilder<'a> {
+impl ExprRewriter for LogicalOverWindowBuilder<'_> {
     fn rewrite_window_function(&mut self, window_func: WindowFunction) -> ExprImpl {
         let dummy = Literal::new(None, window_func.return_type()).into();
         match self.try_rewrite_window_function(window_func) {
@@ -226,7 +226,7 @@ impl<'a> OverWindowProjectBuilder<'a> {
     }
 }
 
-impl<'a> ExprVisitor for OverWindowProjectBuilder<'a> {
+impl ExprVisitor for OverWindowProjectBuilder<'_> {
     fn visit_window_function(&mut self, window_function: &WindowFunction) {
         if let Err(e) = self.try_visit_window_function(window_function) {
             self.error = Some(e);
@@ -278,7 +278,7 @@ impl LogicalOverWindow {
         let rewritten_selected_items = over_window_builder.rewrite_selected_items(select_exprs)?;
 
         for window_func in &window_functions {
-            if window_func.kind.is_rank() && window_func.order_by.sort_exprs.is_empty() {
+            if window_func.kind.is_numbering() && window_func.order_by.sort_exprs.is_empty() {
                 return Err(ErrorCode::InvalidInputSyntax(format!(
                     "window rank function without order by: {:?}",
                     window_func
diff --git a/src/frontend/src/optimizer/plan_node/logical_postgres_query.rs b/src/frontend/src/optimizer/plan_node/logical_postgres_query.rs
index 9082bd86a3f37..d3d793fb8ba01 100644
--- a/src/frontend/src/optimizer/plan_node/logical_postgres_query.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_postgres_query.rs
@@ -103,13 +103,13 @@ impl ToBatch for LogicalPostgresQuery {
 
 impl ToStream for LogicalPostgresQuery {
     fn to_stream(&self, _ctx: &mut ToStreamContext) -> Result<PlanRef> {
-        bail!("file_scan function is not supported in streaming mode")
+        bail!("postgres_query function is not supported in streaming mode")
     }
 
     fn logical_rewrite_for_stream(
         &self,
         _ctx: &mut RewriteStreamContext,
     ) -> Result<(PlanRef, ColIndexMapping)> {
-        bail!("file_scan function is not supported in streaming mode")
+        bail!("postgres_query function is not supported in streaming mode")
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/logical_scan.rs b/src/frontend/src/optimizer/plan_node/logical_scan.rs
index e2aeaa6b9517a..ccb90889cb946 100644
--- a/src/frontend/src/optimizer/plan_node/logical_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_scan.rs
@@ -419,13 +419,13 @@ impl PredicatePushdown for LogicalScan {
             self.clone_with_predicate(predicate.and(self.predicate().clone()))
                 .into()
         } else {
-            return LogicalFilter::create(
+            LogicalFilter::create(
                 self.clone_with_predicate(predicate.and(self.predicate().clone()))
                     .into(),
                 Condition {
                     conjunctions: non_pushable_predicate,
                 },
-            );
+            )
         }
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/logical_source.rs b/src/frontend/src/optimizer/plan_node/logical_source.rs
index 5bbf1a0216411..81dc1128c2a43 100644
--- a/src/frontend/src/optimizer/plan_node/logical_source.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_source.rs
@@ -354,7 +354,7 @@ impl ToStream for LogicalSource {
             }
             SourceNodeKind::CreateMViewOrBatch => {
                 // Create MV on source.
-                // We only check enable_shared_source is true when `CREATE SOURCE`.
+                // We only check streaming_use_shared_source is true when `CREATE SOURCE`.
                 // The value does not affect the behavior of `CREATE MATERIALIZED VIEW` here.
                 let use_shared_source = self.source_catalog().is_some_and(|c| c.info.is_shared());
                 if use_shared_source {
diff --git a/src/frontend/src/optimizer/plan_node/logical_sys_scan.rs b/src/frontend/src/optimizer/plan_node/logical_sys_scan.rs
index a85afa8afa039..cf711f74a039f 100644
--- a/src/frontend/src/optimizer/plan_node/logical_sys_scan.rs
+++ b/src/frontend/src/optimizer/plan_node/logical_sys_scan.rs
@@ -309,13 +309,13 @@ impl PredicatePushdown for LogicalSysScan {
             self.clone_with_predicate(predicate.and(self.predicate().clone()))
                 .into()
         } else {
-            return LogicalFilter::create(
+            LogicalFilter::create(
                 self.clone_with_predicate(predicate.and(self.predicate().clone()))
                     .into(),
                 Condition {
                     conjunctions: non_pushable_predicate,
                 },
-            );
+            )
         }
     }
 }
diff --git a/src/frontend/src/optimizer/plan_node/merge_eq_nodes.rs b/src/frontend/src/optimizer/plan_node/merge_eq_nodes.rs
index 36d0c1bec73ac..b136d3468472f 100644
--- a/src/frontend/src/optimizer/plan_node/merge_eq_nodes.rs
+++ b/src/frontend/src/optimizer/plan_node/merge_eq_nodes.rs
@@ -92,7 +92,7 @@ impl VisitPlan for Counter {
     where
         F: FnMut(&mut Self),
     {
-        if !self.counts.get(&plan.id()).is_some_and(|c| *c > 1) {
+        if self.counts.get(&plan.id()).is_none_or(|c| *c <= 1) {
             f(self);
         }
     }
diff --git a/src/frontend/src/optimizer/plan_node/mod.rs b/src/frontend/src/optimizer/plan_node/mod.rs
index cfcc4f0709569..432475b52809b 100644
--- a/src/frontend/src/optimizer/plan_node/mod.rs
+++ b/src/frontend/src/optimizer/plan_node/mod.rs
@@ -53,6 +53,7 @@ use super::property::{Distribution, FunctionalDependencySet, MonotonicityMap, Or
 use crate::error::{ErrorCode, Result};
 use crate::optimizer::ExpressionSimplifyRewriter;
 use crate::session::current::notice_to_user;
+use crate::utils::PrettySerde;
 
 /// A marker trait for different conventions, used for enforcing type safety.
 ///
@@ -643,6 +644,9 @@ pub trait Explain {
 
     /// Explain the plan node and return a string.
     fn explain_to_string(&self) -> String;
+
+    /// Explain the plan node and return a json string.
+    fn explain_to_json(&self) -> String;
 }
 
 impl Explain for PlanRef {
@@ -665,6 +669,14 @@ impl Explain for PlanRef {
         config.unicode(&mut output, &plan.explain());
         output
     }
+
+    /// Explain the plan node and return a json string.
+    fn explain_to_json(&self) -> String {
+        let plan = reorganize_elements_id(self.clone());
+        let explain_ir = plan.explain();
+        serde_json::to_string_pretty(&PrettySerde(explain_ir))
+            .expect("failed to serialize plan to json")
+    }
 }
 
 pub(crate) fn pretty_config() -> PrettyConfig {
@@ -923,10 +935,14 @@ mod batch_file_scan;
 mod batch_iceberg_scan;
 mod batch_kafka_scan;
 mod batch_postgres_query;
+
+mod batch_mysql_query;
 mod derive;
 mod logical_file_scan;
 mod logical_iceberg_scan;
 mod logical_postgres_query;
+
+mod logical_mysql_query;
 mod stream_cdc_table_scan;
 mod stream_share;
 mod stream_temporal_join;
@@ -949,6 +965,7 @@ pub use batch_limit::BatchLimit;
 pub use batch_log_seq_scan::BatchLogSeqScan;
 pub use batch_lookup_join::BatchLookupJoin;
 pub use batch_max_one_row::BatchMaxOneRow;
+pub use batch_mysql_query::BatchMySqlQuery;
 pub use batch_nested_loop_join::BatchNestedLoopJoin;
 pub use batch_over_window::BatchOverWindow;
 pub use batch_postgres_query::BatchPostgresQuery;
@@ -985,6 +1002,7 @@ pub use logical_kafka_scan::LogicalKafkaScan;
 pub use logical_limit::LogicalLimit;
 pub use logical_max_one_row::LogicalMaxOneRow;
 pub use logical_multi_join::{LogicalMultiJoin, LogicalMultiJoinBuilder};
+pub use logical_mysql_query::LogicalMySqlQuery;
 pub use logical_now::LogicalNow;
 pub use logical_over_window::LogicalOverWindow;
 pub use logical_postgres_query::LogicalPostgresQuery;
@@ -1100,6 +1118,7 @@ macro_rules! for_all_plan_nodes {
             , { Logical, ChangeLog }
             , { Logical, FileScan }
             , { Logical, PostgresQuery }
+            , { Logical, MySqlQuery }
             , { Batch, SimpleAgg }
             , { Batch, HashAgg }
             , { Batch, SortAgg }
@@ -1132,6 +1151,7 @@ macro_rules! for_all_plan_nodes {
             , { Batch, IcebergScan }
             , { Batch, FileScan }
             , { Batch, PostgresQuery }
+            , { Batch, MySqlQuery }
             , { Stream, Project }
             , { Stream, Filter }
             , { Stream, TableScan }
@@ -1214,6 +1234,7 @@ macro_rules! for_logical_plan_nodes {
             , { Logical, ChangeLog }
             , { Logical, FileScan }
             , { Logical, PostgresQuery }
+            , { Logical, MySqlQuery }
         }
     };
 }
@@ -1255,6 +1276,7 @@ macro_rules! for_batch_plan_nodes {
             , { Batch, IcebergScan }
             , { Batch, FileScan }
             , { Batch, PostgresQuery }
+            , { Batch, MySqlQuery }
         }
     };
 }
diff --git a/src/frontend/src/optimizer/plan_node/stream_materialize.rs b/src/frontend/src/optimizer/plan_node/stream_materialize.rs
index 88878667b8411..342bfbedd1825 100644
--- a/src/frontend/src/optimizer/plan_node/stream_materialize.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_materialize.rs
@@ -21,6 +21,7 @@ use pretty_xmlish::{Pretty, XmlNode};
 use risingwave_common::catalog::{
     ColumnCatalog, ConflictBehavior, CreateType, StreamJobStatus, TableId, OBJECT_ID_PLACEHOLDER,
 };
+use risingwave_common::hash::VnodeCount;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 use risingwave_pb::stream_plan::stream_node::PbNodeBody;
@@ -33,7 +34,7 @@ use crate::catalog::table_catalog::{TableCatalog, TableType, TableVersion};
 use crate::error::Result;
 use crate::optimizer::plan_node::derive::derive_pk;
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::plan_node::utils::plan_can_use_backgronud_ddl;
+use crate::optimizer::plan_node::utils::plan_can_use_background_ddl;
 use crate::optimizer::plan_node::{PlanBase, PlanNodeMeta};
 use crate::optimizer::property::{Cardinality, Distribution, Order, RequiredDist};
 use crate::stream_fragmenter::BuildFragmentGraphState;
@@ -88,7 +89,7 @@ impl StreamMaterialize {
 
         let create_type = if matches!(table_type, TableType::MaterializedView)
             && input.ctx().session_ctx().config().background_ddl()
-            && plan_can_use_backgronud_ddl(&input)
+            && plan_can_use_background_ddl(&input)
         {
             CreateType::Background
         } else {
@@ -283,7 +284,7 @@ impl StreamMaterialize {
             created_at_cluster_version: None,
             retention_seconds: retention_seconds.map(|i| i.into()),
             cdc_table_id: None,
-            vnode_count: None, // will be filled in by the meta service later
+            vnode_count: VnodeCount::Placeholder, // will be filled in by the meta service later
         })
     }
 
diff --git a/src/frontend/src/optimizer/plan_node/stream_sink.rs b/src/frontend/src/optimizer/plan_node/stream_sink.rs
index 39cf84fe13705..48dc4dad85c5c 100644
--- a/src/frontend/src/optimizer/plan_node/stream_sink.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_sink.rs
@@ -46,7 +46,7 @@ use super::{generic, ExprRewritable, PlanBase, PlanRef, StreamNode, StreamProjec
 use crate::error::{ErrorCode, Result};
 use crate::expr::{ExprImpl, FunctionCall, InputRef};
 use crate::optimizer::plan_node::expr_visitable::ExprVisitable;
-use crate::optimizer::plan_node::utils::plan_can_use_backgronud_ddl;
+use crate::optimizer::plan_node::utils::plan_can_use_background_ddl;
 use crate::optimizer::plan_node::PlanTreeNodeUnary;
 use crate::optimizer::property::{Distribution, Order, RequiredDist};
 use crate::stream_fragmenter::BuildFragmentGraphState;
@@ -385,7 +385,7 @@ impl StreamSink {
         let input = required_dist.enforce_if_not_satisfies(input, &Order::any())?;
         let distribution_key = input.distribution().dist_column_indices().to_vec();
         let create_type = if input.ctx().session_ctx().config().background_ddl()
-            && plan_can_use_backgronud_ddl(&input)
+            && plan_can_use_background_ddl(&input)
         {
             CreateType::Background
         } else {
diff --git a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
index 7f50d4b27e625..390a141dcb385 100644
--- a/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
+++ b/src/frontend/src/optimizer/plan_node/stream_temporal_join.rs
@@ -42,12 +42,19 @@ pub struct StreamTemporalJoin {
     core: generic::Join<PlanRef>,
     eq_join_predicate: EqJoinPredicate,
     append_only: bool,
+    is_nested_loop: bool,
 }
 
 impl StreamTemporalJoin {
-    pub fn new(core: generic::Join<PlanRef>, eq_join_predicate: EqJoinPredicate) -> Self {
+    pub fn new(
+        core: generic::Join<PlanRef>,
+        eq_join_predicate: EqJoinPredicate,
+        is_nested_loop: bool,
+    ) -> Self {
         assert!(core.join_type == JoinType::Inner || core.join_type == JoinType::LeftOuter);
         let append_only = core.left.append_only();
+        assert!(!is_nested_loop || append_only);
+
         let right = core.right.clone();
         let exchange: &StreamExchange = right
             .as_stream_exchange()
@@ -59,8 +66,16 @@ impl StreamTemporalJoin {
             .expect("should be a stream table scan");
         assert!(matches!(scan.core().as_of, Some(AsOf::ProcessTime)));
 
-        let l2o = core.l2i_col_mapping().composite(&core.i2o_col_mapping());
-        let dist = l2o.rewrite_provided_distribution(core.left.distribution());
+        let dist = if is_nested_loop {
+            // Use right side distribution directly if it's nested loop temporal join.
+            let r2o = core.r2i_col_mapping().composite(&core.i2o_col_mapping());
+            r2o.rewrite_provided_distribution(core.right.distribution())
+        } else {
+            // Use left side distribution directly if it's hash temporal join.
+            // https://github.com/risingwavelabs/risingwave/pull/19201#discussion_r1824031780
+            let l2o = core.l2i_col_mapping().composite(&core.i2o_col_mapping());
+            l2o.rewrite_provided_distribution(core.left.distribution())
+        };
 
         // Use left side watermark directly.
         let watermark_columns = core.i2o_col_mapping().rewrite_bitset(
@@ -89,6 +104,7 @@ impl StreamTemporalJoin {
             core,
             eq_join_predicate,
             append_only,
+            is_nested_loop,
         }
     }
 
@@ -105,6 +121,10 @@ impl StreamTemporalJoin {
         self.append_only
     }
 
+    pub fn is_nested_loop(&self) -> bool {
+        self.eq_join_predicate().has_eq()
+    }
+
     /// Return memo-table catalog and its `pk_indices`.
     /// (`join_key` + `left_pk` + `right_pk`) -> (`right_scan_schema` + `join_key` + `left_pk`)
     ///
@@ -170,6 +190,8 @@ impl Distill for StreamTemporalJoin {
             }),
         ));
 
+        vec.push(("nested_loop", Pretty::debug(&self.is_nested_loop)));
+
         if let Some(ow) = watermark_pretty(self.base.watermark_columns(), self.schema()) {
             vec.push(("output_watermarks", ow));
         }
@@ -196,7 +218,7 @@ impl PlanTreeNodeBinary for StreamTemporalJoin {
         let mut core = self.core.clone();
         core.left = left;
         core.right = right;
-        Self::new(core, self.eq_join_predicate.clone())
+        Self::new(core, self.eq_join_predicate.clone(), self.is_nested_loop)
     }
 }
 
@@ -244,6 +266,7 @@ impl TryToStreamPb for StreamTemporalJoin {
                 memo_table = memo_table.with_id(state.gen_table_id_wrapped());
                 Some(memo_table.to_internal_table_prost())
             },
+            is_nested_loop: self.is_nested_loop,
         }))
     }
 }
@@ -256,7 +279,12 @@ impl ExprRewritable for StreamTemporalJoin {
     fn rewrite_exprs(&self, r: &mut dyn ExprRewriter) -> PlanRef {
         let mut core = self.core.clone();
         core.rewrite_exprs(r);
-        Self::new(core, self.eq_join_predicate.rewrite_exprs(r)).into()
+        Self::new(
+            core,
+            self.eq_join_predicate.rewrite_exprs(r),
+            self.is_nested_loop,
+        )
+        .into()
     }
 }
 
diff --git a/src/frontend/src/optimizer/plan_node/utils.rs b/src/frontend/src/optimizer/plan_node/utils.rs
index 4aefa1e54c882..2433a659bad0a 100644
--- a/src/frontend/src/optimizer/plan_node/utils.rs
+++ b/src/frontend/src/optimizer/plan_node/utils.rs
@@ -27,6 +27,7 @@ use risingwave_common::catalog::{
 use risingwave_common::constants::log_store::v2::{
     KV_LOG_STORE_PREDEFINED_COLUMNS, PK_ORDERING, VNODE_COLUMN_INDEX,
 };
+use risingwave_common::hash::VnodeCount;
 use risingwave_common::util::sort_util::{ColumnOrder, OrderType};
 
 use crate::catalog::table_catalog::TableType;
@@ -137,6 +138,21 @@ impl TableCatalogBuilder {
             Some(w) => w,
             None => FixedBitSet::with_capacity(self.columns.len()),
         };
+
+        // If `dist_key_in_pk` is set, check if it matches with `distribution_key`.
+        // Note that we cannot derive in the opposite direction, because there can be a column
+        // appearing multiple times in the PK.
+        if let Some(dist_key_in_pk) = &self.dist_key_in_pk {
+            let derived_dist_key = dist_key_in_pk
+                .iter()
+                .map(|idx| self.pk[*idx].column_index)
+                .collect_vec();
+            assert_eq!(
+                derived_dist_key, distribution_key,
+                "dist_key mismatch with dist_key_in_pk"
+            );
+        }
+
         TableCatalog {
             id: TableId::placeholder(),
             associated_source_id: None,
@@ -179,7 +195,7 @@ impl TableCatalogBuilder {
             created_at_cluster_version: None,
             retention_seconds: None,
             cdc_table_id: None,
-            vnode_count: None, // will be filled in by the meta service later
+            vnode_count: VnodeCount::Placeholder, // will be filled in by the meta service later
         }
     }
 
@@ -379,9 +395,12 @@ pub fn infer_kv_log_store_table_catalog_inner(
 /// since that plan node maps to `backfill` executor, which supports recovery.
 /// Some other leaf nodes like `StreamValues` do not support recovery, and they
 /// cannot use background ddl.
-pub(crate) fn plan_can_use_backgronud_ddl(plan: &PlanRef) -> bool {
+pub(crate) fn plan_can_use_background_ddl(plan: &PlanRef) -> bool {
     if plan.inputs().is_empty() {
-        if plan.as_stream_source_scan().is_some() {
+        if plan.as_stream_source_scan().is_some()
+            || plan.as_stream_now().is_some()
+            || plan.as_stream_source().is_some()
+        {
             true
         } else if let Some(scan) = plan.as_stream_table_scan() {
             scan.stream_scan_type() == StreamScanType::Backfill
@@ -391,7 +410,7 @@ pub(crate) fn plan_can_use_backgronud_ddl(plan: &PlanRef) -> bool {
         }
     } else {
         assert!(!plan.inputs().is_empty());
-        plan.inputs().iter().all(plan_can_use_backgronud_ddl)
+        plan.inputs().iter().all(plan_can_use_background_ddl)
     }
 }
 
diff --git a/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs b/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs
index e2d042791af5d..4a62fe5aa4f8a 100644
--- a/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs
+++ b/src/frontend/src/optimizer/plan_visitor/plan_correlated_id_finder.rs
@@ -43,7 +43,6 @@ impl PlanCorrelatedIdFinder {
 impl PlanVisitor for PlanCorrelatedIdFinder {
     /// `correlated_input_ref` can only appear in `LogicalProject`, `LogicalFilter`,
     /// `LogicalJoin` or the `filter` clause of `PlanAggCall` of `LogicalAgg` now.
-
     type Result = ();
 
     type DefaultBehavior = impl DefaultBehavior<Self::Result>;
diff --git a/src/frontend/src/optimizer/rule/apply_join_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_join_transpose_rule.rs
index 4ae41ce3bd564..9e6f7582c0590 100644
--- a/src/frontend/src/optimizer/rule/apply_join_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_join_transpose_rule.rs
@@ -24,7 +24,9 @@ use crate::expr::{
     InputRef,
 };
 use crate::optimizer::plan_node::generic::GenericPlanRef;
-use crate::optimizer::plan_node::{LogicalApply, LogicalFilter, LogicalJoin, PlanTreeNodeBinary};
+use crate::optimizer::plan_node::{
+    LogicalApply, LogicalFilter, LogicalJoin, PlanTreeNode, PlanTreeNodeBinary,
+};
 use crate::optimizer::plan_visitor::{ExprCorrelatedIdFinder, PlanCorrelatedIdFinder};
 use crate::optimizer::rule::apply_offset_rewriter::ApplyCorrelatedIndicesConverter;
 use crate::optimizer::PlanRef;
@@ -122,10 +124,14 @@ impl Rule for ApplyJoinTransposeRule {
             return None;
         }
 
-        assert!(
-            join.output_indices_are_trivial(),
-            "ApplyJoinTransposeRule requires the join containing no output indices, so make sure ProjectJoinSeparateRule is always applied before this rule"
-        );
+        // ApplyJoinTransposeRule requires the join containing no output indices, so make sure ProjectJoinSeparateRule is always applied before this rule.
+        // As this rule will be applied until we reach the fixed point, if the join has output indices, apply ProjectJoinSeparateRule first and return is safety.
+        if !join.output_indices_are_trivial() {
+            let new_apply_right = crate::optimizer::rule::ProjectJoinSeparateRule::create()
+                .apply(join.clone().into())
+                .unwrap();
+            return Some(apply.clone_with_inputs(&[apply_left, new_apply_right]));
+        }
 
         let (push_left, push_right) = match join.join_type() {
             // `LeftSemi`, `LeftAnti`, `LeftOuter` can only push to left side if it's right side has
diff --git a/src/frontend/src/optimizer/rule/apply_union_transpose_rule.rs b/src/frontend/src/optimizer/rule/apply_union_transpose_rule.rs
index 0bfe6e268e14e..e66f4c8e11bc8 100644
--- a/src/frontend/src/optimizer/rule/apply_union_transpose_rule.rs
+++ b/src/frontend/src/optimizer/rule/apply_union_transpose_rule.rs
@@ -39,7 +39,6 @@ use crate::optimizer::PlanRef;
 ///   /      \           /      \
 /// Domain   T1        Domain   T2
 /// ```
-
 pub struct ApplyUnionTransposeRule {}
 impl Rule for ApplyUnionTransposeRule {
     fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
diff --git a/src/frontend/src/optimizer/rule/mod.rs b/src/frontend/src/optimizer/rule/mod.rs
index 56d79bf7b408b..7468f1c96524c 100644
--- a/src/frontend/src/optimizer/rule/mod.rs
+++ b/src/frontend/src/optimizer/rule/mod.rs
@@ -161,6 +161,7 @@ mod pull_up_correlated_predicate_agg_rule;
 mod source_to_iceberg_scan_rule;
 mod source_to_kafka_scan_rule;
 mod table_function_to_file_scan_rule;
+mod table_function_to_mysql_query_rule;
 mod table_function_to_postgres_query_rule;
 mod values_extract_project_rule;
 
@@ -169,6 +170,7 @@ pub use pull_up_correlated_predicate_agg_rule::*;
 pub use source_to_iceberg_scan_rule::*;
 pub use source_to_kafka_scan_rule::*;
 pub use table_function_to_file_scan_rule::*;
+pub use table_function_to_mysql_query_rule::*;
 pub use table_function_to_postgres_query_rule::*;
 pub use values_extract_project_rule::*;
 
@@ -234,6 +236,7 @@ macro_rules! for_all_rules {
             , { TableFunctionToProjectSetRule }
             , { TableFunctionToFileScanRule }
             , { TableFunctionToPostgresQueryRule }
+            , { TableFunctionToMySqlQueryRule }
             , { ApplyLimitTransposeRule }
             , { CommonSubExprExtractRule }
             , { BatchProjectMergeRule }
diff --git a/src/frontend/src/optimizer/rule/over_window_split_rule.rs b/src/frontend/src/optimizer/rule/over_window_split_rule.rs
index b8114f85302ca..85510a5ff9746 100644
--- a/src/frontend/src/optimizer/rule/over_window_split_rule.rs
+++ b/src/frontend/src/optimizer/rule/over_window_split_rule.rs
@@ -36,7 +36,7 @@ impl Rule for OverWindowSplitRule {
             .iter()
             .enumerate()
             .map(|(idx, func)| {
-                let func_seq = if func.kind.is_rank() {
+                let func_seq = if func.kind.is_numbering() {
                     rank_func_seq += 1;
                     rank_func_seq
                 } else {
diff --git a/src/frontend/src/optimizer/rule/over_window_to_topn_rule.rs b/src/frontend/src/optimizer/rule/over_window_to_topn_rule.rs
index 76fbb649038a6..10ab630a64e3a 100644
--- a/src/frontend/src/optimizer/rule/over_window_to_topn_rule.rs
+++ b/src/frontend/src/optimizer/rule/over_window_to_topn_rule.rs
@@ -70,7 +70,7 @@ impl Rule for OverWindowToTopNRule {
             return None;
         }
         let window_func = &over_window.window_functions()[0];
-        if !window_func.kind.is_rank() {
+        if !window_func.kind.is_numbering() {
             // Only rank functions can be converted to TopN.
             return None;
         }
diff --git a/src/frontend/src/optimizer/rule/project_join_separate_rule.rs b/src/frontend/src/optimizer/rule/project_join_separate_rule.rs
index 2cbdd143a6669..ce144e341ed24 100644
--- a/src/frontend/src/optimizer/rule/project_join_separate_rule.rs
+++ b/src/frontend/src/optimizer/rule/project_join_separate_rule.rs
@@ -25,7 +25,7 @@ impl ProjectJoinSeparateRule {
 impl Rule for ProjectJoinSeparateRule {
     fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
         let join = plan.as_logical_join()?;
-        if join.is_full_out() {
+        if join.output_indices_are_trivial() {
             None
         } else {
             let (left, right, on, join_type, output_indices) = join.clone().decompose();
diff --git a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
index 4a59dcda785b8..0cd3dfdd5716a 100644
--- a/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
+++ b/src/frontend/src/optimizer/rule/pull_up_correlated_predicate_agg_rule.rs
@@ -59,7 +59,6 @@ use crate::utils::{Condition, IndexSet};
 ///                 |
 ///               Filter
 /// ```
-
 pub struct PullUpCorrelatedPredicateAggRule {}
 impl Rule for PullUpCorrelatedPredicateAggRule {
     fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
diff --git a/src/frontend/src/optimizer/rule/table_function_to_mysql_query_rule.rs b/src/frontend/src/optimizer/rule/table_function_to_mysql_query_rule.rs
new file mode 100644
index 0000000000000..0ad534825790c
--- /dev/null
+++ b/src/frontend/src/optimizer/rule/table_function_to_mysql_query_rule.rs
@@ -0,0 +1,91 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use itertools::Itertools;
+use risingwave_common::catalog::{Field, Schema};
+use risingwave_common::types::{DataType, ScalarImpl};
+use risingwave_common::util::iter_util::ZipEqDebug;
+
+use super::{BoxedRule, Rule};
+use crate::expr::{Expr, TableFunctionType};
+use crate::optimizer::plan_node::generic::GenericPlanRef;
+// use crate::optimizer::plan_node::{LogicalMySqlQuery, LogicalTableFunction};
+use crate::optimizer::plan_node::{LogicalMySqlQuery, LogicalTableFunction};
+use crate::optimizer::PlanRef;
+
+/// Transform a special `TableFunction` (with `MYSQL_QUERY` table function type) into a `LogicalMySqlQuery`
+pub struct TableFunctionToMySqlQueryRule {}
+impl Rule for TableFunctionToMySqlQueryRule {
+    fn apply(&self, plan: PlanRef) -> Option<PlanRef> {
+        let logical_table_function: &LogicalTableFunction = plan.as_logical_table_function()?;
+        if logical_table_function.table_function.function_type != TableFunctionType::MysqlQuery {
+            return None;
+        }
+        assert!(!logical_table_function.with_ordinality);
+        let table_function_return_type = logical_table_function.table_function().return_type();
+
+        if let DataType::Struct(st) = table_function_return_type.clone() {
+            let fields = st
+                .types()
+                .zip_eq_debug(st.names())
+                .map(|(data_type, name)| Field::with_name(data_type.clone(), name.to_string()))
+                .collect_vec();
+
+            let schema = Schema::new(fields);
+
+            assert_eq!(logical_table_function.table_function().args.len(), 6);
+            let mut eval_args = vec![];
+            for arg in &logical_table_function.table_function().args {
+                assert_eq!(arg.return_type(), DataType::Varchar);
+                let value = arg.try_fold_const().unwrap().unwrap();
+                match value {
+                    Some(ScalarImpl::Utf8(s)) => {
+                        eval_args.push(s.to_string());
+                    }
+                    _ => {
+                        unreachable!("must be a varchar")
+                    }
+                }
+            }
+            let hostname = eval_args[0].clone();
+            let port = eval_args[1].clone();
+            let username = eval_args[2].clone();
+            let password = eval_args[3].clone();
+            let database = eval_args[4].clone();
+            let query = eval_args[5].clone();
+
+            Some(
+                LogicalMySqlQuery::new(
+                    logical_table_function.ctx(),
+                    schema,
+                    hostname,
+                    port,
+                    username,
+                    password,
+                    database,
+                    query,
+                )
+                .into(),
+            )
+        } else {
+            unreachable!("TableFunction return type should be struct")
+        }
+    }
+}
+
+impl TableFunctionToMySqlQueryRule {
+    pub fn create() -> BoxedRule {
+        Box::new(TableFunctionToMySqlQueryRule {})
+    }
+}
diff --git a/src/frontend/src/scheduler/distributed/query.rs b/src/frontend/src/scheduler/distributed/query.rs
index d169874d5b3f2..2d40328bab400 100644
--- a/src/frontend/src/scheduler/distributed/query.rs
+++ b/src/frontend/src/scheduler/distributed/query.rs
@@ -473,7 +473,7 @@ pub(crate) mod tests {
         ColumnCatalog, ColumnDesc, ConflictBehavior, CreateType, StreamJobStatus,
         DEFAULT_SUPER_USER_ID,
     };
-    use risingwave_common::hash::{VirtualNode, WorkerSlotId, WorkerSlotMapping};
+    use risingwave_common::hash::{VirtualNode, VnodeCount, WorkerSlotId, WorkerSlotMapping};
     use risingwave_common::types::DataType;
     use risingwave_pb::common::worker_node::Property;
     use risingwave_pb::common::{HostAddress, WorkerNode, WorkerType};
@@ -589,7 +589,7 @@ pub(crate) mod tests {
             initialized_at_cluster_version: None,
             created_at_cluster_version: None,
             cdc_table_id: None,
-            vnode_count: Some(vnode_count),
+            vnode_count: VnodeCount::set(vnode_count),
         };
         let batch_plan_node: PlanRef = LogicalScan::create(
             "".to_string(),
diff --git a/src/frontend/src/scheduler/plan_fragmenter.rs b/src/frontend/src/scheduler/plan_fragmenter.rs
index 9bd1851383af6..90984750bc460 100644
--- a/src/frontend/src/scheduler/plan_fragmenter.rs
+++ b/src/frontend/src/scheduler/plan_fragmenter.rs
@@ -1228,8 +1228,21 @@ fn derive_partitions(
     table_desc: &TableDesc,
     vnode_mapping: &WorkerSlotMapping,
 ) -> SchedulerResult<HashMap<WorkerSlotId, TablePartitionInfo>> {
+    let vnode_mapping = if table_desc.vnode_count != vnode_mapping.len() {
+        // The vnode count mismatch occurs only in special cases where a hash-distributed fragment
+        // contains singleton internal tables. e.g., the state table of `Source` executors.
+        // In this case, we reduce the vnode mapping to a single vnode as only `SINGLETON_VNODE` is used.
+        assert!(
+            table_desc.vnode_count == 1,
+            "fragment vnode count {} does not match table vnode count {}",
+            vnode_mapping.len(),
+            table_desc.vnode_count,
+        );
+        &WorkerSlotMapping::new_single(vnode_mapping.iter().next().unwrap())
+    } else {
+        vnode_mapping
+    };
     let vnode_count = vnode_mapping.len();
-    assert_eq!(vnode_count, table_desc.vnode_count);
 
     let mut partitions: HashMap<WorkerSlotId, (BitmapBuilder, Vec<_>)> = HashMap::new();
 
diff --git a/src/frontend/src/scheduler/streaming_manager.rs b/src/frontend/src/scheduler/streaming_manager.rs
index dfe04acaa24dd..6e9c07eb8f98e 100644
--- a/src/frontend/src/scheduler/streaming_manager.rs
+++ b/src/frontend/src/scheduler/streaming_manager.rs
@@ -92,7 +92,7 @@ pub struct StreamingJobGuard<'a> {
     tracker: &'a StreamingJobTracker,
 }
 
-impl<'a> Drop for StreamingJobGuard<'a> {
+impl Drop for StreamingJobGuard<'_> {
     fn drop(&mut self) {
         self.tracker.delete_job(&self.task_id);
     }
diff --git a/src/frontend/src/session/cursor_manager.rs b/src/frontend/src/session/cursor_manager.rs
index 4115da5f3025e..f8464120a531d 100644
--- a/src/frontend/src/session/cursor_manager.rs
+++ b/src/frontend/src/session/cursor_manager.rs
@@ -22,7 +22,6 @@ use std::time::Instant;
 
 use anyhow::anyhow;
 use bytes::Bytes;
-use fixedbitset::FixedBitSet;
 use futures::StreamExt;
 use pgwire::pg_field_descriptor::PgFieldDescriptor;
 use pgwire::pg_response::StatementType;
@@ -31,20 +30,21 @@ use risingwave_common::catalog::Field;
 use risingwave_common::error::BoxedError;
 use risingwave_common::session_config::QueryMode;
 use risingwave_common::types::DataType;
+use risingwave_common::util::sort_util::ColumnOrder;
 use risingwave_hummock_sdk::HummockVersionId;
 use risingwave_sqlparser::ast::{Ident, ObjectName, Statement};
 
 use super::SessionImpl;
 use crate::catalog::subscription_catalog::SubscriptionCatalog;
 use crate::catalog::TableId;
-use crate::error::{ErrorCode, Result};
-use crate::handler::declare_cursor::{
-    create_chunk_stream_for_cursor, create_stream_for_cursor_stmt,
+use crate::error::{ErrorCode, Result, RwError};
+use crate::handler::declare_cursor::create_chunk_stream_for_cursor;
+use crate::handler::query::{
+    gen_batch_plan_by_statement, gen_batch_plan_fragmenter, BatchQueryPlanResult,
 };
-use crate::handler::query::{gen_batch_plan_fragmenter, BatchQueryPlanResult};
 use crate::handler::util::{
-    convert_logstore_u64_to_unix_millis, gen_query_from_table_name, pg_value_format, to_pg_field,
-    DataChunkToRowSetAdapter, StaticSessionData,
+    convert_logstore_u64_to_unix_millis, gen_query_from_table_name_order_by, pg_value_format,
+    to_pg_field, DataChunkToRowSetAdapter, StaticSessionData,
 };
 use crate::handler::HandlerArgs;
 use crate::monitor::{CursorMetrics, PeriodicCursorMetrics};
@@ -52,7 +52,9 @@ use crate::optimizer::plan_node::{generic, BatchLogSeqScan};
 use crate::optimizer::property::{Order, RequiredDist};
 use crate::optimizer::PlanRoot;
 use crate::scheduler::{DistributedQueryStream, LocalQueryStream};
-use crate::{OptimizerContext, OptimizerContextRef, PgResponseStream, PlanRef, TableCatalog};
+use crate::{
+    Binder, OptimizerContext, OptimizerContextRef, PgResponseStream, PlanRef, TableCatalog,
+};
 
 pub enum CursorDataChunkStream {
     LocalDataChunk(Option<LocalQueryStream>),
@@ -111,18 +113,18 @@ impl Cursor {
     pub async fn next(
         &mut self,
         count: u32,
-        handle_args: HandlerArgs,
+        handler_args: HandlerArgs,
         formats: &Vec<Format>,
         timeout_seconds: Option<u64>,
     ) -> Result<(Vec<Row>, Vec<PgFieldDescriptor>)> {
         match self {
             Cursor::Subscription(cursor) => cursor
-                .next(count, handle_args, formats, timeout_seconds)
+                .next(count, handler_args, formats, timeout_seconds)
                 .await
                 .inspect_err(|_| cursor.cursor_metrics.subscription_cursor_error_count.inc()),
             Cursor::Query(cursor) => {
                 cursor
-                    .next(count, formats, handle_args, timeout_seconds)
+                    .next(count, formats, handler_args, timeout_seconds)
                     .await
             }
         }
@@ -168,13 +170,13 @@ impl QueryCursor {
         &mut self,
         count: u32,
         formats: &Vec<Format>,
-        handle_args: HandlerArgs,
+        handler_args: HandlerArgs,
         timeout_seconds: Option<u64>,
     ) -> Result<(Vec<Row>, Vec<PgFieldDescriptor>)> {
         // `FETCH NEXT` is equivalent to `FETCH 1`.
         // min with 100 to avoid allocating too many memory at once.
         let timeout_instant = timeout_seconds.map(|s| Instant::now() + Duration::from_secs(s));
-        let session = handle_args.session;
+        let session = handler_args.session;
         let mut ans = Vec::with_capacity(std::cmp::min(100, count) as usize);
         let mut cur = 0;
         let desc = self.fields.iter().map(to_pg_field).collect();
@@ -273,11 +275,11 @@ impl SubscriptionCursor {
         start_timestamp: Option<u64>,
         subscription: Arc<SubscriptionCatalog>,
         dependent_table_id: TableId,
-        handle_args: &HandlerArgs,
+        handler_args: &HandlerArgs,
         cursor_metrics: Arc<CursorMetrics>,
     ) -> Result<Self> {
         let (state, fields) = if let Some(start_timestamp) = start_timestamp {
-            let table_catalog = handle_args.session.get_table_by_id(&dependent_table_id)?;
+            let table_catalog = handler_args.session.get_table_by_id(&dependent_table_id)?;
             let fields = table_catalog
                 .columns
                 .iter()
@@ -298,8 +300,8 @@ impl SubscriptionCursor {
             //
             // TODO: is this the right behavior? Should we delay the query stream initiation till the first fetch?
             let (chunk_stream, fields, init_query_timer) =
-                Self::initiate_query(None, &dependent_table_id, handle_args.clone()).await?;
-            let pinned_epoch = handle_args
+                Self::initiate_query(None, &dependent_table_id, handler_args.clone()).await?;
+            let pinned_epoch = handler_args
                 .session
                 .env
                 .hummock_snapshot_manager
@@ -341,7 +343,7 @@ impl SubscriptionCursor {
 
     async fn next_row(
         &mut self,
-        handle_args: &HandlerArgs,
+        handler_args: &HandlerArgs,
         formats: &Vec<Format>,
     ) -> Result<Option<Row>> {
         loop {
@@ -357,7 +359,7 @@ impl SubscriptionCursor {
                         *seek_timestamp,
                         &self.dependent_table_id,
                         *expected_timestamp,
-                        handle_args.clone(),
+                        handler_args.clone(),
                         &self.subscription,
                     ) {
                         Ok((Some(rw_timestamp), expected_timestamp)) => {
@@ -365,7 +367,7 @@ impl SubscriptionCursor {
                                 Self::initiate_query(
                                     Some(rw_timestamp),
                                     &self.dependent_table_id,
-                                    handle_args.clone(),
+                                    handler_args.clone(),
                                 )
                                 .await?;
                             Self::init_row_stream(
@@ -373,7 +375,7 @@ impl SubscriptionCursor {
                                 formats,
                                 &from_snapshot,
                                 &fields,
-                                handle_args.session.clone(),
+                                handler_args.session.clone(),
                             );
 
                             self.cursor_need_drop_time = Instant::now()
@@ -411,7 +413,7 @@ impl SubscriptionCursor {
                     init_query_timer,
                 } => {
                     let session_data = StaticSessionData {
-                        timezone: handle_args.session.config().timezone(),
+                        timezone: handler_args.session.config().timezone(),
                     };
                     let from_snapshot = *from_snapshot;
                     let rw_timestamp = *rw_timestamp;
@@ -471,7 +473,7 @@ impl SubscriptionCursor {
     pub async fn next(
         &mut self,
         count: u32,
-        handle_args: HandlerArgs,
+        handler_args: HandlerArgs,
         formats: &Vec<Format>,
         timeout_seconds: Option<u64>,
     ) -> Result<(Vec<Row>, Vec<PgFieldDescriptor>)> {
@@ -483,7 +485,7 @@ impl SubscriptionCursor {
             .into());
         }
 
-        let session = &handle_args.session;
+        let session = &handler_args.session;
         let mut ans = Vec::with_capacity(std::cmp::min(100, count) as usize);
         let mut cur = 0;
         if let State::Fetch {
@@ -502,7 +504,7 @@ impl SubscriptionCursor {
         }
         while cur < count {
             let fetch_cursor_timer = Instant::now();
-            let row = self.next_row(&handle_args, formats).await?;
+            let row = self.next_row(&handler_args, formats).await?;
             self.cursor_metrics
                 .subscription_cursor_fetch_duration
                 .with_label_values(&[&self.subscription.name])
@@ -553,10 +555,10 @@ impl SubscriptionCursor {
         seek_timestamp: u64,
         table_id: &TableId,
         expected_timestamp: Option<u64>,
-        handle_args: HandlerArgs,
+        handler_args: HandlerArgs,
         dependent_subscription: &SubscriptionCatalog,
     ) -> Result<(Option<u64>, Option<u64>)> {
-        let session = handle_args.session;
+        let session = handler_args.session;
         // Test subscription existence
         session.get_subscription_by_schema_id_name(
             dependent_subscription.schema_id,
@@ -580,16 +582,50 @@ impl SubscriptionCursor {
         Ok((new_epochs.get(0).cloned(), new_epochs.get(1).cloned()))
     }
 
-    async fn initiate_query(
+    pub fn gen_batch_plan_result(&self, handler_args: HandlerArgs) -> Result<BatchQueryPlanResult> {
+        match self.state {
+            // Only used to return generated plans, so rw_timestamp are meaningless
+            State::InitLogStoreQuery { .. } => Self::init_batch_plan_for_subscription_cursor(
+                Some(0),
+                &self.dependent_table_id,
+                handler_args,
+            ),
+            State::Fetch {
+                from_snapshot,
+                rw_timestamp,
+                ..
+            } => {
+                if from_snapshot {
+                    Self::init_batch_plan_for_subscription_cursor(
+                        None,
+                        &self.dependent_table_id,
+                        handler_args,
+                    )
+                } else {
+                    Self::init_batch_plan_for_subscription_cursor(
+                        Some(rw_timestamp),
+                        &self.dependent_table_id,
+                        handler_args,
+                    )
+                }
+            }
+            State::Invalid => Err(ErrorCode::InternalError(
+                "Cursor is in invalid state. Please close and re-create the cursor.".to_string(),
+            )
+            .into()),
+        }
+    }
+
+    fn init_batch_plan_for_subscription_cursor(
         rw_timestamp: Option<u64>,
         dependent_table_id: &TableId,
-        handle_args: HandlerArgs,
-    ) -> Result<(CursorDataChunkStream, Vec<Field>, Instant)> {
-        let session = handle_args.clone().session;
+        handler_args: HandlerArgs,
+    ) -> Result<BatchQueryPlanResult> {
+        let session = handler_args.clone().session;
         let table_catalog = session.get_table_by_id(dependent_table_id)?;
-        let init_query_timer = Instant::now();
-        let (chunk_stream, fields) = if let Some(rw_timestamp) = rw_timestamp {
-            let context = OptimizerContext::from_handler_args(handle_args);
+        let pks = table_catalog.pk();
+        let context = OptimizerContext::from_handler_args(handler_args.clone());
+        if let Some(rw_timestamp) = rw_timestamp {
             let version_id = {
                 let version = session.env.hummock_snapshot_manager.acquire();
                 let version = version.version();
@@ -602,26 +638,58 @@ impl SubscriptionCursor {
                 }
                 version.id
             };
-            let plan_fragmenter_result = gen_batch_plan_fragmenter(
+            Self::create_batch_plan_for_cursor(
+                &table_catalog,
                 &session,
-                Self::create_batch_plan_for_cursor(
-                    &table_catalog,
-                    &session,
-                    context.into(),
-                    rw_timestamp,
-                    rw_timestamp,
-                    version_id,
-                )?,
-            )?;
-            create_chunk_stream_for_cursor(session, plan_fragmenter_result).await?
+                context.into(),
+                rw_timestamp,
+                rw_timestamp,
+                version_id,
+                pks,
+            )
         } else {
             let subscription_from_table_name =
                 ObjectName(vec![Ident::from(table_catalog.name.as_ref())]);
-            let query_stmt = Statement::Query(Box::new(gen_query_from_table_name(
+            let pk_names = pks
+                .iter()
+                .map(|f| {
+                    Ok::<String, RwError>(
+                        table_catalog
+                            .columns
+                            .get(f.column_index)
+                            .ok_or_else(|| {
+                                anyhow!(
+                                    "columns not find in table schema, index is {:?}",
+                                    f.column_index
+                                )
+                            })?
+                            .name()
+                            .to_string(),
+                    )
+                })
+                .collect::<Result<Vec<_>>>()?;
+            let query_stmt = Statement::Query(Box::new(gen_query_from_table_name_order_by(
                 subscription_from_table_name,
+                pk_names,
             )));
-            create_stream_for_cursor_stmt(handle_args, query_stmt).await?
-        };
+            gen_batch_plan_by_statement(&session, context.into(), query_stmt)
+        }
+    }
+
+    async fn initiate_query(
+        rw_timestamp: Option<u64>,
+        dependent_table_id: &TableId,
+        handler_args: HandlerArgs,
+    ) -> Result<(CursorDataChunkStream, Vec<Field>, Instant)> {
+        let init_query_timer = Instant::now();
+        let plan_result = Self::init_batch_plan_for_subscription_cursor(
+            rw_timestamp,
+            dependent_table_id,
+            handler_args.clone(),
+        )?;
+        let plan_fragmenter_result = gen_batch_plan_fragmenter(&handler_args.session, plan_result)?;
+        let (chunk_stream, fields) =
+            create_chunk_stream_for_cursor(handler_args.session, plan_fragmenter_result).await?;
         Ok((
             chunk_stream,
             Self::build_desc(fields, rw_timestamp.is_none()),
@@ -687,38 +755,54 @@ impl SubscriptionCursor {
         old_epoch: u64,
         new_epoch: u64,
         version_id: HummockVersionId,
+        pks: &[ColumnOrder],
     ) -> Result<BatchQueryPlanResult> {
-        let out_col_idx = table_catalog
+        // pk + all column without hidden
+        let output_col_idx = table_catalog
             .columns
             .iter()
             .enumerate()
-            .filter(|(_, v)| !v.is_hidden)
-            .map(|(i, _)| i)
+            .filter_map(|(index, v)| {
+                if !v.is_hidden || table_catalog.pk.iter().any(|pk| pk.column_index == index) {
+                    Some(index)
+                } else {
+                    None
+                }
+            })
+            .collect::<Vec<_>>();
+        let output_col_idx_with_out_hidden = output_col_idx
+            .iter()
+            .filter(|index| !table_catalog.columns[**index].is_hidden)
+            .cloned()
             .collect::<Vec<_>>();
         let core = generic::LogScan::new(
             table_catalog.name.clone(),
-            out_col_idx,
+            output_col_idx_with_out_hidden,
+            output_col_idx,
             Rc::new(table_catalog.table_desc()),
             context,
             old_epoch,
             new_epoch,
             version_id,
         );
+
         let batch_log_seq_scan = BatchLogSeqScan::new(core);
-        let schema = batch_log_seq_scan
-            .core()
-            .schema_without_table_name()
-            .clone();
-        let out_fields = FixedBitSet::from_iter(0..schema.len());
-        let out_names = batch_log_seq_scan.core().column_names();
+
+        let out_fields = batch_log_seq_scan.core().out_fields();
+        let out_names = batch_log_seq_scan.core().column_names_without_hidden();
+
+        // order by pk, so don't need to sort
+        let order = Order::new(pks.to_vec());
+
         // Here we just need a plan_root to call the method, only out_fields and out_names will be used
         let plan_root = PlanRoot::new_with_batch_plan(
             PlanRef::from(batch_log_seq_scan.clone()),
             RequiredDist::single(),
-            Order::default(),
+            order,
             out_fields,
             out_names,
         );
+        let schema = plan_root.schema().clone();
         let (batch_log_seq_scan, query_mode) = match session.config().query_mode() {
             QueryMode::Auto => (plan_root.gen_batch_local_plan()?, QueryMode::Local),
             QueryMode::Local => (plan_root.gen_batch_local_plan()?, QueryMode::Local),
@@ -789,7 +873,7 @@ impl CursorManager {
         start_timestamp: Option<u64>,
         dependent_table_id: TableId,
         subscription: Arc<SubscriptionCatalog>,
-        handle_args: &HandlerArgs,
+        handler_args: &HandlerArgs,
     ) -> Result<()> {
         let create_cursor_timer = Instant::now();
         let subscription_name = subscription.name.clone();
@@ -798,7 +882,7 @@ impl CursorManager {
             start_timestamp,
             subscription,
             dependent_table_id,
-            handle_args,
+            handler_args,
             self.cursor_metrics.clone(),
         )
         .await?;
@@ -870,16 +954,16 @@ impl CursorManager {
         &self,
         cursor_name: String,
         count: u32,
-        handle_args: HandlerArgs,
+        handler_args: HandlerArgs,
         formats: &Vec<Format>,
         timeout_seconds: Option<u64>,
     ) -> Result<(Vec<Row>, Vec<PgFieldDescriptor>)> {
         if let Some(cursor) = self.cursor_map.lock().await.get_mut(&cursor_name) {
             cursor
-                .next(count, handle_args, formats, timeout_seconds)
+                .next(count, handler_args, formats, timeout_seconds)
                 .await
         } else {
-            Err(ErrorCode::ItemNotFound(format!("Cannot find cursor `{}`", cursor_name)).into())
+            Err(ErrorCode::InternalError(format!("Cannot find cursor `{}`", cursor_name)).into())
         }
     }
 
@@ -887,7 +971,7 @@ impl CursorManager {
         if let Some(cursor) = self.cursor_map.lock().await.get_mut(&cursor_name) {
             Ok(cursor.get_fields())
         } else {
-            Err(ErrorCode::ItemNotFound(format!("Cannot find cursor `{}`", cursor_name)).into())
+            Err(ErrorCode::InternalError(format!("Cannot find cursor `{}`", cursor_name)).into())
         }
     }
 
@@ -940,4 +1024,22 @@ impl CursorManager {
                 }
             });
     }
+
+    pub async fn gen_batch_plan_with_subscription_cursor(
+        &self,
+        cursor_name: ObjectName,
+        handler_args: HandlerArgs,
+    ) -> Result<BatchQueryPlanResult> {
+        let session = handler_args.session.clone();
+        let db_name = session.database();
+        let (_, cursor_name) = Binder::resolve_schema_qualified_name(db_name, cursor_name.clone())?;
+        match self.cursor_map.lock().await.get(&cursor_name).ok_or_else(|| {
+            ErrorCode::InternalError(format!("Cannot find cursor `{}`", cursor_name))
+        })? {
+            Cursor::Subscription(cursor) => {
+                cursor.gen_batch_plan_result(handler_args.clone())
+            },
+            Cursor::Query(_) => Err(ErrorCode::InternalError("The plan of the cursor is the same as the query statement of the as when it was created.".to_string()).into()),
+        }
+    }
 }
diff --git a/src/frontend/src/test_utils.rs b/src/frontend/src/test_utils.rs
index ff1c92227716c..d94b1dd2652d6 100644
--- a/src/frontend/src/test_utils.rs
+++ b/src/frontend/src/test_utils.rs
@@ -28,6 +28,7 @@ use risingwave_common::catalog::{
     FunctionId, IndexId, TableId, DEFAULT_DATABASE_NAME, DEFAULT_SCHEMA_NAME, DEFAULT_SUPER_USER,
     DEFAULT_SUPER_USER_ID, NON_RESERVED_USER_ID, PG_CATALOG_SCHEMA_NAME, RW_CATALOG_SCHEMA_NAME,
 };
+use risingwave_common::hash::{VirtualNode, VnodeCount, VnodeCountCompat};
 use risingwave_common::session_config::SessionConfig;
 use risingwave_common::system_param::reader::SystemParamsReader;
 use risingwave_common::util::cluster_limit::ClusterLimit;
@@ -43,8 +44,8 @@ use risingwave_pb::catalog::{
 use risingwave_pb::common::WorkerNode;
 use risingwave_pb::ddl_service::alter_owner_request::Object;
 use risingwave_pb::ddl_service::{
-    alter_name_request, alter_set_schema_request, create_connection_request, DdlProgress,
-    PbTableJobType, ReplaceTablePlan, TableJobType,
+    alter_name_request, alter_set_schema_request, alter_swap_rename_request,
+    create_connection_request, DdlProgress, PbTableJobType, ReplaceTablePlan, TableJobType,
 };
 use risingwave_pb::hummock::write_limits::WriteLimit;
 use risingwave_pb::hummock::{
@@ -183,6 +184,7 @@ impl LocalFrontend {
         res
     }
 
+    /// Creates a new session
     pub fn session_ref(&self) -> Arc<SessionImpl> {
         self.session_user_ref(
             DEFAULT_DATABASE_NAME.to_string(),
@@ -280,6 +282,7 @@ impl CatalogWriter for MockCatalogWriter {
     ) -> Result<()> {
         table.id = self.gen_id();
         table.stream_job_status = PbStreamJobStatus::Created as _;
+        table.maybe_vnode_count = VnodeCount::for_test().to_protobuf();
         self.catalog.write().create_table(&table);
         self.add_table_or_source_id(table.id, table.schema_id, table.database_id);
         self.hummock_snapshot_manager
@@ -319,6 +322,7 @@ impl CatalogWriter for MockCatalogWriter {
         _job_type: TableJobType,
     ) -> Result<()> {
         table.stream_job_status = PbStreamJobStatus::Created as _;
+        assert_eq!(table.vnode_count(), VirtualNode::COUNT_FOR_TEST);
         self.catalog.write().update_table(&table);
         Ok(())
     }
@@ -352,6 +356,7 @@ impl CatalogWriter for MockCatalogWriter {
     ) -> Result<()> {
         index_table.id = self.gen_id();
         index_table.stream_job_status = PbStreamJobStatus::Created as _;
+        index_table.maybe_vnode_count = VnodeCount::for_test().to_protobuf();
         self.catalog.write().create_table(&index_table);
         self.add_table_or_index_id(
             index_table.id,
@@ -640,6 +645,10 @@ impl CatalogWriter for MockCatalogWriter {
     ) -> Result<()> {
         todo!()
     }
+
+    async fn alter_swap_rename(&self, _object: alter_swap_rename_request::Object) -> Result<()> {
+        todo!()
+    }
 }
 
 impl MockCatalogWriter {
@@ -917,7 +926,7 @@ pub struct MockFrontendMetaClient {}
 impl FrontendMetaClient for MockFrontendMetaClient {
     async fn try_unregister(&self) {}
 
-    async fn flush(&self, _checkpoint: bool) -> RpcResult<HummockVersionId> {
+    async fn flush(&self, _database_id: DatabaseId) -> RpcResult<HummockVersionId> {
         Ok(INVALID_VERSION_ID)
     }
 
diff --git a/src/frontend/src/utils/condition.rs b/src/frontend/src/utils/condition.rs
index 5bb53a84ae410..f3f9738e2cae1 100644
--- a/src/frontend/src/utils/condition.rs
+++ b/src/frontend/src/utils/condition.rs
@@ -663,14 +663,14 @@ impl Condition {
         new_conds: &ScalarImpl,
         eq_conds: &[Option<ScalarImpl>],
     ) -> bool {
-        return !eq_conds.is_empty()
+        !eq_conds.is_empty()
             && eq_conds.iter().all(|l| {
                 if let Some(l) = l {
                     l != new_conds
                 } else {
                     true
                 }
-            });
+            })
     }
 
     fn merge_lower_bound_conjunctions(lb: Vec<Bound<ScalarImpl>>) -> Bound<ScalarImpl> {
diff --git a/src/frontend/src/utils/index_set.rs b/src/frontend/src/utils/index_set.rs
index 000fb2ef924a1..e34d2f1f66cb4 100644
--- a/src/frontend/src/utils/index_set.rs
+++ b/src/frontend/src/utils/index_set.rs
@@ -106,7 +106,7 @@ impl BitAnd for IndexSet {
     }
 }
 
-impl<'a> BitAnd for &'a IndexSet {
+impl BitAnd for &IndexSet {
     type Output = IndexSet;
 
     fn bitand(self, rhs: Self) -> Self::Output {
@@ -126,7 +126,7 @@ impl BitOr for IndexSet {
     }
 }
 
-impl<'a> BitOr for &'a IndexSet {
+impl BitOr for &IndexSet {
     type Output = IndexSet;
 
     fn bitor(self, rhs: Self) -> Self::Output {
diff --git a/src/frontend/src/utils/mod.rs b/src/frontend/src/utils/mod.rs
index ca2a90e4e38e7..c1556f30b1caf 100644
--- a/src/frontend/src/utils/mod.rs
+++ b/src/frontend/src/utils/mod.rs
@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+mod pretty_serde;
+pub use pretty_serde::PrettySerde;
 mod column_index_mapping;
 use std::any::Any;
 use std::hash::{Hash, Hasher};
diff --git a/src/frontend/src/utils/pretty_serde.rs b/src/frontend/src/utils/pretty_serde.rs
new file mode 100644
index 0000000000000..705267c3163b6
--- /dev/null
+++ b/src/frontend/src/utils/pretty_serde.rs
@@ -0,0 +1,104 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! @kwannoel:
+//! This module implements Serde for the Pretty struct. Why not implement it directly on our plan nodes?
+//! That's because Pretty already summarizes the fields that are important to us.
+//! You can see that when `explain()` is called, we directly return the `Pretty` struct.
+//! The _proper way_ to do this would be to create a new data structure that plan nodes get converted into,
+//! and then implement `Serialize` and `Deserialize` on that data structure (including to `Pretty`).
+//! But that's a lot of refactoring work.
+//! So we just wrap `Pretty` in a newtype and implement `Serialize` on that,
+//! since it's a good enough intermediate representation.
+
+use std::collections::BTreeMap;
+
+use pretty_xmlish::Pretty;
+use serde::ser::{SerializeSeq, SerializeStruct};
+use serde::{Serialize, Serializer};
+
+pub struct PrettySerde<'a>(pub Pretty<'a>);
+
+impl Serialize for PrettySerde<'_> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        match &self.0 {
+            Pretty::Text(text) => serializer.serialize_str(text.as_ref()),
+
+            Pretty::Record(node) => {
+                let mut state = serializer.serialize_struct("XmlNode", 3)?;
+                state.serialize_field("name", node.name.as_ref())?;
+                state.serialize_field(
+                    "fields",
+                    &node
+                        .fields
+                        .iter()
+                        .map(|(k, v)| (k.as_ref(), PrettySerde(v.clone())))
+                        .collect::<BTreeMap<_, _>>(),
+                )?;
+                state.serialize_field(
+                    "children",
+                    &node
+                        .children
+                        .iter()
+                        .map(|c| PrettySerde(c.clone()))
+                        .collect::<Vec<_>>(),
+                )?;
+                state.end()
+            }
+
+            Pretty::Array(elements) => {
+                let mut seq = serializer.serialize_seq(Some(elements.len()))?;
+                for element in elements {
+                    seq.serialize_element(&PrettySerde((*element).clone()))?;
+                }
+                seq.end()
+            }
+
+            Pretty::Linearized(inner, size) => {
+                let mut state = serializer.serialize_struct("Linearized", 2)?;
+                state.serialize_field("inner", &PrettySerde((**inner).clone()))?;
+                state.serialize_field("size", size)?;
+                state.end()
+            }
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fmt::Debug;
+
+    use expect_test::{expect, Expect};
+
+    use super::*;
+
+    fn check(actual: impl Debug, expect: Expect) {
+        let actual = format!("{:#?}", actual);
+        expect.assert_eq(&actual);
+    }
+
+    #[test]
+    fn test_pretty_serde() {
+        let pretty = Pretty::childless_record("root", vec![("a", Pretty::Text("1".into()))]);
+        let pretty_serde = PrettySerde(pretty);
+        let serialized = serde_json::to_string(&pretty_serde).unwrap();
+        check(
+            serialized,
+            expect![[r#""{\"name\":\"root\",\"fields\":{\"a\":\"1\"},\"children\":[]}""#]],
+        );
+    }
+}
diff --git a/src/frontend/src/utils/with_options.rs b/src/frontend/src/utils/with_options.rs
index e306103c02e39..9d61021dab4fe 100644
--- a/src/frontend/src/utils/with_options.rs
+++ b/src/frontend/src/utils/with_options.rs
@@ -16,7 +16,7 @@ use std::collections::BTreeMap;
 use std::num::NonZeroU32;
 
 use risingwave_connector::source::kafka::private_link::{
-    insert_privatelink_broker_rewrite_map, CONNECTION_NAME_KEY, PRIVATELINK_ENDPOINT_KEY,
+    insert_privatelink_broker_rewrite_map, PRIVATELINK_ENDPOINT_KEY,
 };
 pub use risingwave_connector::WithOptionsSecResolved;
 use risingwave_connector::WithPropertiesExt;
@@ -28,7 +28,6 @@ use risingwave_sqlparser::ast::{
 };
 
 use super::OverwriteOptions;
-use crate::catalog::connection_catalog::resolve_private_link_connection;
 use crate::catalog::ConnectionId;
 use crate::error::{ErrorCode, Result as RwResult, RwError};
 use crate::session::SessionImpl;
@@ -186,8 +185,6 @@ pub(crate) fn resolve_secret_ref_in_with_options(
 
 pub(crate) fn resolve_privatelink_in_with_option(
     with_options: &mut WithOptions,
-    schema_name: &Option<String>,
-    session: &SessionImpl,
 ) -> RwResult<Option<ConnectionId>> {
     let is_kafka = with_options.is_kafka_connector();
     let privatelink_endpoint = with_options.remove(PRIVATELINK_ENDPOINT_KEY);
@@ -201,28 +198,8 @@ pub(crate) fn resolve_privatelink_in_with_option(
         }
         insert_privatelink_broker_rewrite_map(with_options.inner_mut(), None, Some(endpoint))
             .map_err(RwError::from)?;
-        return Ok(None);
     }
-
-    let connection_name = with_options
-        .remove(CONNECTION_NAME_KEY)
-        .map(|s| s.to_lowercase());
-    let connection_id = match connection_name {
-        Some(connection_name) => {
-            let connection = session
-                .get_connection_by_name(schema_name.clone(), &connection_name)
-                .map_err(|_| ErrorCode::ItemNotFound(connection_name))?;
-            if !is_kafka {
-                return Err(RwError::from(ErrorCode::ProtocolError(
-                    "Connection is only supported in kafka connector".to_string(),
-                )));
-            }
-            resolve_private_link_connection(&connection, with_options.inner_mut())?;
-            Some(connection.id)
-        }
-        None => None,
-    };
-    Ok(connection_id)
+    Ok(None)
 }
 
 impl TryFrom<&[SqlOption]> for WithOptions {
diff --git a/src/jni_core/src/lib.rs b/src/jni_core/src/lib.rs
index 3f776c3a2e984..d700d979896cc 100644
--- a/src/jni_core/src/lib.rs
+++ b/src/jni_core/src/lib.rs
@@ -118,7 +118,7 @@ pub struct SliceGuard<'env, 'array> {
     slice: &'array [u8],
 }
 
-impl<'env, 'array> Deref for SliceGuard<'env, 'array> {
+impl Deref for SliceGuard<'_, '_> {
     type Target = [u8];
 
     fn deref(&self) -> &Self::Target {
@@ -132,7 +132,7 @@ pub struct Pointer<'a, T> {
     _phantom: PhantomData<&'a T>,
 }
 
-impl<'a, T> Default for Pointer<'a, T> {
+impl<T> Default for Pointer<'_, T> {
     fn default() -> Self {
         Self {
             pointer: 0,
@@ -187,7 +187,7 @@ impl<'a> Deref for EnvParam<'a> {
     }
 }
 
-impl<'a> DerefMut for EnvParam<'a> {
+impl DerefMut for EnvParam<'_> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.env
     }
@@ -306,7 +306,7 @@ impl JavaBindingIterator<'static> {
     }
 }
 
-impl<'a> Deref for JavaBindingIterator<'a> {
+impl Deref for JavaBindingIterator<'_> {
     type Target = OwnedRow;
 
     fn deref(&self) -> &Self::Target {
diff --git a/src/meta/Cargo.toml b/src/meta/Cargo.toml
index 4773a88802184..16a303609699b 100644
--- a/src/meta/Cargo.toml
+++ b/src/meta/Cargo.toml
@@ -42,7 +42,10 @@ jsonbb = { workspace = true }
 maplit = "1.0.2"
 memcomparable = { version = "0.2" }
 mime_guess = "2"
-notify = { version = "6", default-features = false, features = ["macos_fsevent"] }
+moka = { version = "0.12.3", features = ["future"] }
+notify = { version = "7", default-features = false, features = [
+    "macos_fsevent",
+] }
 num-integer = "0.1"
 num-traits = "0.2"
 otlp-embedded = { workspace = true }
@@ -84,14 +87,14 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
 tokio-retry = "0.3"
 tokio-stream = { workspace = true }
 tonic = { workspace = true }
-tower = { version = "0.4", features = ["util", "load-shed"] }
+tower = { version = "0.5", features = ["util", "load-shed"] }
 tracing = "0.1"
 url = "2"
 uuid = { version = "1", features = ["v4"] }
 
 [target.'cfg(not(madsim))'.dependencies]
 axum = { workspace = true }
-tower-http = { version = "0.5", features = [
+tower-http = { version = "0.6", features = [
     "add-extension",
     "cors",
     "fs",
diff --git a/src/meta/model/migration/Cargo.toml b/src/meta/model/migration/Cargo.toml
index a54753da16fbb..9d9fbd5292d25 100644
--- a/src/meta/model/migration/Cargo.toml
+++ b/src/meta/model/migration/Cargo.toml
@@ -15,11 +15,9 @@ normal = ["workspace-hack"]
 
 [dependencies]
 async-std = { version = "1", features = ["attributes", "tokio1"] }
+easy-ext = "1"
 sea-orm = { workspace = true }
+sea-orm-migration = { workspace = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 uuid = { version = "1", features = ["v4"] }
-
-[dependencies.sea-orm-migration]
-version = "0.12.14"
-features = ["sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", "runtime-tokio-native-tls", "with-uuid"]
diff --git a/src/meta/model/migration/README.md b/src/meta/model/migration/README.md
index d0253be5c05d4..e354d556ed6de 100644
--- a/src/meta/model/migration/README.md
+++ b/src/meta/model/migration/README.md
@@ -54,3 +54,9 @@
 ## Adding a migration
 
 - Add a new column to some catalogs. You can checkout the migration [m20240617_070131_index_column_properties.rs](src/m20240617_070131_index_column_properties.rs) as a reference.
+
+### Special notes on MySQL backend
+
+MySQL data types typically have stricter length limits compared to those in PostgreSQL or SQLite. For example, the `VARCHAR`, `TEXT`, `BLOB`, and `BINARY` data types have a maximum length of 65,535 bytes.
+
+If you need to store more data, e.g., SQL definition, UDF body, protobuf-encoded internal data, etc., please **avoid** using the built-in constructors like `ColumnDef::text` or `ColumnDef::blob`, but use the extensions defined in [`./src/utils.rs`](./src/utils.rs) instead.
diff --git a/src/meta/model/migration/src/lib.rs b/src/meta/model/migration/src/lib.rs
index 08aed0484408c..b84a29891eee3 100644
--- a/src/meta/model/migration/src/lib.rs
+++ b/src/meta/model/migration/src/lib.rs
@@ -22,6 +22,9 @@ mod m20240726_063833_auto_schema_change;
 mod m20240806_143329_add_rate_limit_to_source_catalog;
 mod m20240820_081248_add_time_travel_per_table_epoch;
 mod m20240911_083152_variable_vnode_count;
+mod m20241016_065621_hummock_gc_history;
+mod m20241025_062548_singleton_vnode_count;
+mod utils;
 
 pub struct Migrator;
 
@@ -81,6 +84,8 @@ impl MigratorTrait for Migrator {
             Box::new(m20240806_143329_add_rate_limit_to_source_catalog::Migration),
             Box::new(m20240820_081248_add_time_travel_per_table_epoch::Migration),
             Box::new(m20240911_083152_variable_vnode_count::Migration),
+            Box::new(m20241016_065621_hummock_gc_history::Migration),
+            Box::new(m20241025_062548_singleton_vnode_count::Migration),
         ]
     }
 }
diff --git a/src/meta/model/migration/src/m20230908_072257_init.rs b/src/meta/model/migration/src/m20230908_072257_init.rs
index 7840c40cd8f30..38fba91011569 100644
--- a/src/meta/model/migration/src/m20230908_072257_init.rs
+++ b/src/meta/model/migration/src/m20230908_072257_init.rs
@@ -1,6 +1,7 @@
 use sea_orm_migration::prelude::{Index as MigrationIndex, Table as MigrationTable, *};
 
 use crate::sea_orm::{DatabaseBackend, DbBackend, Statement};
+use crate::utils::ColumnDefExt;
 use crate::{assert_not_has_tables, drop_tables};
 
 #[derive(DeriveMigrationName)]
@@ -140,7 +141,7 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(User::CanCreateDb).boolean().not_null())
                     .col(ColumnDef::new(User::CanCreateUser).boolean().not_null())
                     .col(ColumnDef::new(User::CanLogin).boolean().not_null())
-                    .col(ColumnDef::new(User::AuthInfo).binary())
+                    .col(ColumnDef::new(User::AuthInfo).rw_binary(manager))
                     .to_owned(),
             )
             .await?;
@@ -378,10 +379,14 @@ impl MigrationTrait for Migration {
                     )
                     .col(
                         ColumnDef::new(Fragment::StreamNode)
-                            .blob(BlobSize::Long)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(Fragment::VnodeMapping)
+                            .rw_binary(manager)
                             .not_null(),
                     )
-                    .col(ColumnDef::new(Fragment::VnodeMapping).binary().not_null())
                     .col(ColumnDef::new(Fragment::StateTableIds).json_binary())
                     .col(ColumnDef::new(Fragment::UpstreamFragmentId).json_binary())
                     .foreign_key(
@@ -407,12 +412,16 @@ impl MigrationTrait for Migration {
                     )
                     .col(ColumnDef::new(Actor::FragmentId).integer().not_null())
                     .col(ColumnDef::new(Actor::Status).string().not_null())
-                    .col(ColumnDef::new(Actor::Splits).binary())
+                    .col(ColumnDef::new(Actor::Splits).rw_binary(manager))
                     .col(ColumnDef::new(Actor::ParallelUnitId).integer().not_null())
                     .col(ColumnDef::new(Actor::WorkerId).integer().not_null())
                     .col(ColumnDef::new(Actor::UpstreamActorIds).json_binary())
-                    .col(ColumnDef::new(Actor::VnodeBitmap).binary())
-                    .col(ColumnDef::new(Actor::ExprContext).binary().not_null())
+                    .col(ColumnDef::new(Actor::VnodeBitmap).rw_binary(manager))
+                    .col(
+                        ColumnDef::new(Actor::ExprContext)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .foreign_key(
                         &mut ForeignKey::create()
                             .name("FK_actor_fragment_id")
@@ -454,7 +463,7 @@ impl MigrationTrait for Migration {
                             .json_binary()
                             .not_null(),
                     )
-                    .col(ColumnDef::new(ActorDispatcher::HashMapping).binary())
+                    .col(ColumnDef::new(ActorDispatcher::HashMapping).rw_binary(manager))
                     .col(
                         ColumnDef::new(ActorDispatcher::DispatcherId)
                             .integer()
@@ -495,7 +504,11 @@ impl MigrationTrait for Migration {
                             .primary_key(),
                     )
                     .col(ColumnDef::new(Connection::Name).string().not_null())
-                    .col(ColumnDef::new(Connection::Info).binary().not_null())
+                    .col(
+                        ColumnDef::new(Connection::Info)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .foreign_key(
                         &mut ForeignKey::create()
                             .name("FK_connection_object_id")
@@ -514,16 +527,28 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(Source::SourceId).integer().primary_key())
                     .col(ColumnDef::new(Source::Name).string().not_null())
                     .col(ColumnDef::new(Source::RowIdIndex).integer())
-                    .col(ColumnDef::new(Source::Columns).binary().not_null())
+                    .col(
+                        ColumnDef::new(Source::Columns)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .col(ColumnDef::new(Source::PkColumnIds).json_binary().not_null())
                     .col(
                         ColumnDef::new(Source::WithProperties)
                             .json_binary()
                             .not_null(),
                     )
-                    .col(ColumnDef::new(Source::Definition).text().not_null())
-                    .col(ColumnDef::new(Source::SourceInfo).binary())
-                    .col(ColumnDef::new(Source::WatermarkDescs).binary().not_null())
+                    .col(
+                        ColumnDef::new(Source::Definition)
+                            .rw_long_text(manager)
+                            .not_null(),
+                    )
+                    .col(ColumnDef::new(Source::SourceInfo).rw_binary(manager))
+                    .col(
+                        ColumnDef::new(Source::WatermarkDescs)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .col(ColumnDef::new(Source::OptionalAssociatedTableId).integer())
                     .col(ColumnDef::new(Source::ConnectionId).integer())
                     .col(ColumnDef::new(Source::Version).big_integer().not_null())
@@ -562,8 +587,8 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(Table::OptionalAssociatedSourceId).integer())
                     .col(ColumnDef::new(Table::TableType).string().not_null())
                     .col(ColumnDef::new(Table::BelongsToJobId).integer())
-                    .col(ColumnDef::new(Table::Columns).binary().not_null())
-                    .col(ColumnDef::new(Table::Pk).binary().not_null())
+                    .col(ColumnDef::new(Table::Columns).rw_binary(manager).not_null())
+                    .col(ColumnDef::new(Table::Pk).rw_binary(manager).not_null())
                     .col(
                         ColumnDef::new(Table::DistributionKey)
                             .json_binary()
@@ -575,7 +600,11 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(Table::VnodeColIndex).integer())
                     .col(ColumnDef::new(Table::RowIdIndex).integer())
                     .col(ColumnDef::new(Table::ValueIndices).json_binary().not_null())
-                    .col(ColumnDef::new(Table::Definition).text().not_null())
+                    .col(
+                        ColumnDef::new(Table::Definition)
+                            .rw_long_text(manager)
+                            .not_null(),
+                    )
                     .col(
                         ColumnDef::new(Table::HandlePkConflictBehavior)
                             .string()
@@ -593,14 +622,14 @@ impl MigrationTrait for Migration {
                     )
                     .col(ColumnDef::new(Table::DistKeyInPk).json_binary().not_null())
                     .col(ColumnDef::new(Table::DmlFragmentId).integer())
-                    .col(ColumnDef::new(Table::Cardinality).binary())
+                    .col(ColumnDef::new(Table::Cardinality).rw_binary(manager))
                     .col(
                         ColumnDef::new(Table::CleanedByWatermark)
                             .boolean()
                             .not_null(),
                     )
                     .col(ColumnDef::new(Table::Description).string())
-                    .col(ColumnDef::new(Table::Version).binary())
+                    .col(ColumnDef::new(Table::Version).rw_binary(manager))
                     .col(ColumnDef::new(Table::RetentionSeconds).integer())
                     .col(
                         ColumnDef::new(Table::IncomingSinks)
@@ -655,8 +684,8 @@ impl MigrationTrait for Migration {
                     .table(Sink::Table)
                     .col(ColumnDef::new(Sink::SinkId).integer().primary_key())
                     .col(ColumnDef::new(Sink::Name).string().not_null())
-                    .col(ColumnDef::new(Sink::Columns).binary().not_null())
-                    .col(ColumnDef::new(Sink::PlanPk).binary().not_null())
+                    .col(ColumnDef::new(Sink::Columns).rw_binary(manager).not_null())
+                    .col(ColumnDef::new(Sink::PlanPk).rw_binary(manager).not_null())
                     .col(
                         ColumnDef::new(Sink::DistributionKey)
                             .json_binary()
@@ -665,11 +694,15 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(Sink::DownstreamPk).json_binary().not_null())
                     .col(ColumnDef::new(Sink::SinkType).string().not_null())
                     .col(ColumnDef::new(Sink::Properties).json_binary().not_null())
-                    .col(ColumnDef::new(Sink::Definition).text().not_null())
+                    .col(
+                        ColumnDef::new(Sink::Definition)
+                            .rw_long_text(manager)
+                            .not_null(),
+                    )
                     .col(ColumnDef::new(Sink::ConnectionId).integer())
                     .col(ColumnDef::new(Sink::DbName).string().not_null())
                     .col(ColumnDef::new(Sink::SinkFromName).string().not_null())
-                    .col(ColumnDef::new(Sink::SinkFormatDesc).binary())
+                    .col(ColumnDef::new(Sink::SinkFormatDesc).rw_binary(manager))
                     .col(ColumnDef::new(Sink::TargetTable).integer())
                     .foreign_key(
                         &mut ForeignKey::create()
@@ -703,8 +736,12 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(View::ViewId).integer().primary_key())
                     .col(ColumnDef::new(View::Name).string().not_null())
                     .col(ColumnDef::new(View::Properties).json_binary().not_null())
-                    .col(ColumnDef::new(View::Definition).text().not_null())
-                    .col(ColumnDef::new(View::Columns).binary().not_null())
+                    .col(
+                        ColumnDef::new(View::Definition)
+                            .rw_long_text(manager)
+                            .not_null(),
+                    )
+                    .col(ColumnDef::new(View::Columns).rw_binary(manager).not_null())
                     .foreign_key(
                         &mut ForeignKey::create()
                             .name("FK_view_object_id")
@@ -724,7 +761,11 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(Index::Name).string().not_null())
                     .col(ColumnDef::new(Index::IndexTableId).integer().not_null())
                     .col(ColumnDef::new(Index::PrimaryTableId).integer().not_null())
-                    .col(ColumnDef::new(Index::IndexItems).binary().not_null())
+                    .col(
+                        ColumnDef::new(Index::IndexItems)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .col(ColumnDef::new(Index::IndexColumnsLen).integer().not_null())
                     .foreign_key(
                         &mut ForeignKey::create()
@@ -760,13 +801,22 @@ impl MigrationTrait for Migration {
                     .col(ColumnDef::new(Function::FunctionId).integer().primary_key())
                     .col(ColumnDef::new(Function::Name).string().not_null())
                     .col(ColumnDef::new(Function::ArgNames).string().not_null())
-                    .col(ColumnDef::new(Function::ArgTypes).binary().not_null())
-                    .col(ColumnDef::new(Function::ReturnType).binary().not_null())
+                    .col(
+                        ColumnDef::new(Function::ArgTypes)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(Function::ReturnType)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .col(ColumnDef::new(Function::Language).string().not_null())
                     .col(ColumnDef::new(Function::Link).string())
                     .col(ColumnDef::new(Function::Identifier).string())
-                    .col(ColumnDef::new(Function::Body).string())
-                    .col(ColumnDef::new(Function::CompressedBinary).string())
+                    .col(ColumnDef::new(Function::Body).rw_long_text(manager))
+                    // XXX: should this be binary type instead?
+                    .col(ColumnDef::new(Function::CompressedBinary).rw_long_text(manager))
                     .col(ColumnDef::new(Function::Kind).string().not_null())
                     .col(
                         ColumnDef::new(Function::AlwaysRetryOnNetworkError)
diff --git a/src/meta/model/migration/src/m20231008_020431_hummock.rs b/src/meta/model/migration/src/m20231008_020431_hummock.rs
index 42fe0c424f99c..ea499d67182e3 100644
--- a/src/meta/model/migration/src/m20231008_020431_hummock.rs
+++ b/src/meta/model/migration/src/m20231008_020431_hummock.rs
@@ -1,5 +1,6 @@
 use sea_orm_migration::prelude::*;
 
+use crate::utils::ColumnDefExt;
 use crate::{assert_not_has_tables, drop_tables};
 
 #[derive(DeriveMigrationName)]
@@ -32,7 +33,7 @@ impl MigrationTrait for Migration {
                     )
                     .col(
                         ColumnDef::new(CompactionTask::Task)
-                            .blob(BlobSize::Long)
+                            .rw_binary(manager)
                             .not_null(),
                     )
                     .col(
@@ -54,7 +55,7 @@ impl MigrationTrait for Migration {
                             .not_null()
                             .primary_key(),
                     )
-                    .col(ColumnDef::new(CompactionConfig::Config).blob(BlobSize::Long))
+                    .col(ColumnDef::new(CompactionConfig::Config).rw_binary(manager))
                     .to_owned(),
             )
             .await?;
@@ -69,7 +70,7 @@ impl MigrationTrait for Migration {
                             .not_null()
                             .primary_key(),
                     )
-                    .col(ColumnDef::new(CompactionStatus::Status).blob(BlobSize::Long))
+                    .col(ColumnDef::new(CompactionStatus::Status).rw_binary(manager))
                     .to_owned(),
             )
             .await?;
@@ -142,7 +143,7 @@ impl MigrationTrait for Migration {
                             .boolean()
                             .not_null(),
                     )
-                    .col(ColumnDef::new(HummockVersionDelta::FullVersionDelta).blob(BlobSize::Long))
+                    .col(ColumnDef::new(HummockVersionDelta::FullVersionDelta).rw_binary(manager))
                     .to_owned(),
             )
             .await?;
diff --git a/src/meta/model/migration/src/m20240304_074901_subscription.rs b/src/meta/model/migration/src/m20240304_074901_subscription.rs
index 244c6abe11604..f759d303b25a9 100644
--- a/src/meta/model/migration/src/m20240304_074901_subscription.rs
+++ b/src/meta/model/migration/src/m20240304_074901_subscription.rs
@@ -1,5 +1,6 @@
 use sea_orm_migration::prelude::{Table as MigrationTable, *};
 
+use crate::utils::ColumnDefExt;
 use crate::{assert_not_has_tables, drop_tables};
 
 #[derive(DeriveMigrationName)]
@@ -19,8 +20,16 @@ impl MigrationTrait for Migration {
                             .primary_key(),
                     )
                     .col(ColumnDef::new(Subscription::Name).string().not_null())
-                    .col(ColumnDef::new(Subscription::Columns).binary().not_null())
-                    .col(ColumnDef::new(Subscription::PlanPk).binary().not_null())
+                    .col(
+                        ColumnDef::new(Subscription::Columns)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(Subscription::PlanPk)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .col(
                         ColumnDef::new(Subscription::DistributionKey)
                             .json_binary()
@@ -31,7 +40,11 @@ impl MigrationTrait for Migration {
                             .json_binary()
                             .not_null(),
                     )
-                    .col(ColumnDef::new(Subscription::Definition).string().not_null())
+                    .col(
+                        ColumnDef::new(Subscription::Definition)
+                            .rw_long_text(manager)
+                            .not_null(),
+                    )
                     .col(
                         ColumnDef::new(Subscription::SubscriptionFromName)
                             .string()
diff --git a/src/meta/model/migration/src/m20240506_112555_subscription_partial_ckpt.rs b/src/meta/model/migration/src/m20240506_112555_subscription_partial_ckpt.rs
index 715be0e7cd23f..e82f7aa59d1ab 100644
--- a/src/meta/model/migration/src/m20240506_112555_subscription_partial_ckpt.rs
+++ b/src/meta/model/migration/src/m20240506_112555_subscription_partial_ckpt.rs
@@ -1,6 +1,7 @@
 use sea_orm_migration::prelude::{Table as MigrationTable, *};
 
 use crate::drop_tables;
+use crate::utils::ColumnDefExt;
 
 #[derive(DeriveMigrationName)]
 pub struct Migration;
@@ -20,7 +21,11 @@ impl MigrationTrait for Migration {
                             .primary_key(),
                     )
                     .col(ColumnDef::new(Subscription::Name).string().not_null())
-                    .col(ColumnDef::new(Subscription::Definition).string().not_null())
+                    .col(
+                        ColumnDef::new(Subscription::Definition)
+                            .rw_long_text(manager)
+                            .not_null(),
+                    )
                     .col(
                         ColumnDef::new(Subscription::RetentionSeconds)
                             .string()
@@ -66,9 +71,21 @@ impl MigrationTrait for Migration {
                             .primary_key(),
                     )
                     .col(ColumnDef::new(Subscription::Name).string().not_null())
-                    .col(ColumnDef::new(Subscription::Definition).string().not_null())
-                    .col(ColumnDef::new(Subscription::Columns).binary().not_null())
-                    .col(ColumnDef::new(Subscription::PlanPk).binary().not_null())
+                    .col(
+                        ColumnDef::new(Subscription::Definition)
+                            .rw_long_text(manager)
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(Subscription::Columns)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
+                    .col(
+                        ColumnDef::new(Subscription::PlanPk)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .col(
                         ColumnDef::new(Subscription::DistributionKey)
                             .json_binary()
diff --git a/src/meta/model/migration/src/m20240525_090457_secret.rs b/src/meta/model/migration/src/m20240525_090457_secret.rs
index ed23085c66574..734c0a6d1fd61 100644
--- a/src/meta/model/migration/src/m20240525_090457_secret.rs
+++ b/src/meta/model/migration/src/m20240525_090457_secret.rs
@@ -1,5 +1,6 @@
 use sea_orm_migration::prelude::{Table as MigrationTable, *};
 
+use crate::utils::ColumnDefExt;
 use crate::{assert_not_has_tables, drop_tables};
 
 #[derive(DeriveMigrationName)]
@@ -21,7 +22,7 @@ impl MigrationTrait for Migration {
                             .primary_key(),
                     )
                     .col(ColumnDef::new(Secret::Name).string().not_null())
-                    .col(ColumnDef::new(Secret::Value).binary().not_null())
+                    .col(ColumnDef::new(Secret::Value).rw_binary(manager).not_null())
                     .foreign_key(
                         &mut ForeignKey::create()
                             .name("FK_secret_object_id")
@@ -42,7 +43,7 @@ impl MigrationTrait for Migration {
             .alter_table(
                 MigrationTable::alter()
                     .table(Sink::Table)
-                    .add_column(ColumnDef::new(Sink::SecretRef).binary())
+                    .add_column(ColumnDef::new(Sink::SecretRef).rw_binary(manager))
                     .to_owned(),
             )
             .await?;
@@ -52,7 +53,7 @@ impl MigrationTrait for Migration {
             .alter_table(
                 MigrationTable::alter()
                     .table(Source::Table)
-                    .add_column(ColumnDef::new(Source::SecretRef).binary())
+                    .add_column(ColumnDef::new(Source::SecretRef).rw_binary(manager))
                     .to_owned(),
             )
             .await?;
diff --git a/src/meta/model/migration/src/m20240617_070131_index_column_properties.rs b/src/meta/model/migration/src/m20240617_070131_index_column_properties.rs
index daff755da999f..f8f113e7974db 100644
--- a/src/meta/model/migration/src/m20240617_070131_index_column_properties.rs
+++ b/src/meta/model/migration/src/m20240617_070131_index_column_properties.rs
@@ -1,5 +1,7 @@
 use sea_orm_migration::prelude::*;
 
+use crate::utils::ColumnDefExt;
+
 #[derive(DeriveMigrationName)]
 pub struct Migration;
 
@@ -10,7 +12,7 @@ impl MigrationTrait for Migration {
             .alter_table(
                 Table::alter()
                     .table(Index::Table)
-                    .add_column(ColumnDef::new(Index::IndexColumnProperties).binary())
+                    .add_column(ColumnDef::new(Index::IndexColumnProperties).rw_binary(manager))
                     .to_owned(),
             )
             .await
diff --git a/src/meta/model/migration/src/m20240617_071625_sink_into_table_column.rs b/src/meta/model/migration/src/m20240617_071625_sink_into_table_column.rs
index 4904dd3623245..c4e01c3bcd5be 100644
--- a/src/meta/model/migration/src/m20240617_071625_sink_into_table_column.rs
+++ b/src/meta/model/migration/src/m20240617_071625_sink_into_table_column.rs
@@ -1,5 +1,6 @@
 use sea_orm_migration::prelude::{Table as MigrationTable, *};
 
+use crate::utils::ColumnDefExt;
 use crate::SubQueryStatement::SelectStatement;
 
 #[derive(DeriveMigrationName)]
@@ -12,7 +13,7 @@ impl MigrationTrait for Migration {
             .alter_table(
                 MigrationTable::alter()
                     .table(Sink::Table)
-                    .add_column(ColumnDef::new(Sink::OriginalTargetColumns).binary())
+                    .add_column(ColumnDef::new(Sink::OriginalTargetColumns).rw_binary(manager))
                     .to_owned(),
             )
             .await?;
diff --git a/src/meta/model/migration/src/m20240618_072634_function_compressed_binary.rs b/src/meta/model/migration/src/m20240618_072634_function_compressed_binary.rs
index 6b4ef6157bcf6..92d27da1e1385 100644
--- a/src/meta/model/migration/src/m20240618_072634_function_compressed_binary.rs
+++ b/src/meta/model/migration/src/m20240618_072634_function_compressed_binary.rs
@@ -1,6 +1,7 @@
 use sea_orm_migration::prelude::*;
 
 use crate::sea_orm::{DatabaseBackend, DbBackend, Statement};
+use crate::utils::ColumnDefExt;
 
 #[derive(DeriveMigrationName)]
 pub struct Migration;
@@ -19,7 +20,7 @@ impl MigrationTrait for Migration {
                         Table::alter()
                             .table(Function::Table)
                             .modify_column(
-                                ColumnDef::new(Function::CompressedBinary).blob(BlobSize::Medium),
+                                ColumnDef::new(Function::CompressedBinary).rw_binary(manager),
                             )
                             .to_owned(),
                     )
diff --git a/src/meta/model/migration/src/m20240630_131430_remove_parallel_unit.rs b/src/meta/model/migration/src/m20240630_131430_remove_parallel_unit.rs
index 5acc8dc1a6165..dabcb8d3daa79 100644
--- a/src/meta/model/migration/src/m20240630_131430_remove_parallel_unit.rs
+++ b/src/meta/model/migration/src/m20240630_131430_remove_parallel_unit.rs
@@ -2,6 +2,7 @@ use sea_orm_migration::prelude::*;
 use serde::{Deserialize, Serialize};
 
 use crate::sea_orm::{FromJsonQueryResult, FromQueryResult, Statement};
+use crate::utils::ColumnDefExt;
 
 #[derive(DeriveMigrationName)]
 pub struct Migration;
@@ -75,7 +76,11 @@ impl MigrationTrait for Migration {
             .alter_table(
                 Table::alter()
                     .table(Fragment::Table)
-                    .add_column(ColumnDef::new(Fragment::VnodeMapping).binary().not_null())
+                    .add_column(
+                        ColumnDef::new(Fragment::VnodeMapping)
+                            .rw_binary(manager)
+                            .not_null(),
+                    )
                     .to_owned(),
             )
             .await?;
diff --git a/src/meta/model/migration/src/m20240701_060504_hummock_time_travel.rs b/src/meta/model/migration/src/m20240701_060504_hummock_time_travel.rs
index 7dec44913dc8f..619537078e28e 100644
--- a/src/meta/model/migration/src/m20240701_060504_hummock_time_travel.rs
+++ b/src/meta/model/migration/src/m20240701_060504_hummock_time_travel.rs
@@ -1,6 +1,7 @@
 use sea_orm_migration::prelude::*;
 
 use crate::drop_tables;
+use crate::utils::ColumnDefExt;
 
 #[derive(DeriveMigrationName)]
 pub struct Migration;
@@ -26,7 +27,7 @@ impl MigrationTrait for Migration {
                     )
                     .col(
                         ColumnDef::new(HummockSstableInfo::SstableInfo)
-                            .blob(BlobSize::Long)
+                            .rw_binary(manager)
                             .null(),
                     )
                     .to_owned(),
@@ -46,7 +47,7 @@ impl MigrationTrait for Migration {
                     )
                     .col(
                         ColumnDef::new(HummockTimeTravelVersion::Version)
-                            .blob(BlobSize::Long)
+                            .rw_binary(manager)
                             .null(),
                     )
                     .to_owned(),
@@ -66,7 +67,7 @@ impl MigrationTrait for Migration {
                     )
                     .col(
                         ColumnDef::new(HummockTimeTravelDelta::VersionDelta)
-                            .blob(BlobSize::Long)
+                            .rw_binary(manager)
                             .null(),
                     )
                     .to_owned(),
diff --git a/src/meta/model/migration/src/m20241016_065621_hummock_gc_history.rs b/src/meta/model/migration/src/m20241016_065621_hummock_gc_history.rs
new file mode 100644
index 0000000000000..79254ed8f6661
--- /dev/null
+++ b/src/meta/model/migration/src/m20241016_065621_hummock_gc_history.rs
@@ -0,0 +1,51 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        manager
+            .create_table(
+                Table::create()
+                    .table(HummockGcHistory::Table)
+                    .if_not_exists()
+                    .col(
+                        ColumnDef::new(HummockGcHistory::ObjectId)
+                            .big_integer()
+                            .not_null()
+                            .primary_key(),
+                    )
+                    .col(
+                        ColumnDef::new(HummockGcHistory::MarkDeleteAt)
+                            .date_time()
+                            .not_null(),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+        manager
+            .create_index(
+                Index::create()
+                    .table(HummockGcHistory::Table)
+                    .name("idx_hummock_gc_history_mark_delete_at")
+                    .col(HummockGcHistory::MarkDeleteAt)
+                    .to_owned(),
+            )
+            .await?;
+        Ok(())
+    }
+
+    async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        crate::drop_tables!(manager, HummockGcHistory);
+        Ok(())
+    }
+}
+
+#[derive(DeriveIden)]
+enum HummockGcHistory {
+    Table,
+    ObjectId,
+    MarkDeleteAt,
+}
diff --git a/src/meta/model/migration/src/m20241025_062548_singleton_vnode_count.rs b/src/meta/model/migration/src/m20241025_062548_singleton_vnode_count.rs
new file mode 100644
index 0000000000000..be5b1fef90fdf
--- /dev/null
+++ b/src/meta/model/migration/src/m20241025_062548_singleton_vnode_count.rs
@@ -0,0 +1,64 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        // Hack for unifying the test for different backends.
+        let json_is_empty_array = |col| {
+            Expr::col(col)
+                .cast_as(Alias::new("char(2)"))
+                .eq(Expr::value("[]"))
+        };
+
+        // Fill vnode count with 1 for singleton tables.
+        manager
+            .exec_stmt(
+                UpdateStatement::new()
+                    .table(Table::Table)
+                    .values([(Table::VnodeCount, Expr::value(1))])
+                    .and_where(json_is_empty_array(Table::DistributionKey))
+                    .and_where(json_is_empty_array(Table::DistKeyInPk))
+                    .and_where(Expr::col(Table::VnodeColIndex).is_null())
+                    .to_owned(),
+            )
+            .await?;
+
+        // Fill vnode count with 1 for singleton fragments.
+        manager
+            .exec_stmt(
+                UpdateStatement::new()
+                    .table(Fragment::Table)
+                    .values([(Fragment::VnodeCount, Expr::value(1))])
+                    .and_where(Expr::col(Fragment::DistributionType).eq(Expr::value("SINGLE")))
+                    .to_owned(),
+            )
+            .await?;
+
+        Ok(())
+    }
+
+    async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
+        Err(DbErr::Migration(
+            "cannot rollback singleton vnode count migration".to_owned(),
+        ))?
+    }
+}
+
+#[derive(DeriveIden)]
+enum Fragment {
+    Table,
+    VnodeCount,
+    DistributionType,
+}
+
+#[derive(DeriveIden)]
+enum Table {
+    Table,
+    VnodeCount,
+    DistributionKey,
+    DistKeyInPk,
+    VnodeColIndex,
+}
diff --git a/src/meta/model/migration/src/utils.rs b/src/meta/model/migration/src/utils.rs
new file mode 100644
index 0000000000000..7d3eb1fbb3ae2
--- /dev/null
+++ b/src/meta/model/migration/src/utils.rs
@@ -0,0 +1,29 @@
+use sea_orm::DatabaseBackend;
+use sea_orm_migration::prelude::*;
+
+#[easy_ext::ext(ColumnDefExt)]
+impl ColumnDef {
+    /// Set column type as `longblob` for MySQL, `bytea` for Postgres, and `blob` for Sqlite.
+    ///
+    /// Should be preferred over [`binary`](ColumnDef::binary) or [`blob`](ColumnDef::blob) for large binary fields,
+    /// typically the fields wrapping protobuf or other serialized data. Otherwise, MySQL will return an error
+    /// when the length exceeds 65535 bytes.
+    pub fn rw_binary(&mut self, manager: &SchemaManager) -> &mut Self {
+        match manager.get_database_backend() {
+            DatabaseBackend::MySql => self.custom(extension::mysql::MySqlType::LongBlob),
+            DatabaseBackend::Postgres | DatabaseBackend::Sqlite => self.blob(),
+        }
+    }
+
+    /// Set column type as `longtext` for MySQL, and `text` for Postgres and Sqlite.
+    ///
+    /// Should be preferred over [`text`](ColumnDef::text) or [`string`](ColumnDef::string) for large text fields,
+    /// typically user-specified contents like UDF body or SQL definition. Otherwise, MySQL will return an error
+    /// when the length exceeds 65535 bytes.
+    pub fn rw_long_text(&mut self, manager: &SchemaManager) -> &mut Self {
+        match manager.get_database_backend() {
+            DatabaseBackend::MySql => self.custom(Alias::new("longtext")),
+            DatabaseBackend::Postgres | DatabaseBackend::Sqlite => self.text(),
+        }
+    }
+}
diff --git a/src/meta/model/src/actor.rs b/src/meta/model/src/actor.rs
index cbbbca543679a..2278d7e9fd373 100644
--- a/src/meta/model/src/actor.rs
+++ b/src/meta/model/src/actor.rs
@@ -21,7 +21,7 @@ use crate::{
 };
 
 #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum ActorStatus {
     #[sea_orm(string_value = "INACTIVE")]
     Inactive,
diff --git a/src/meta/model/src/actor_dispatcher.rs b/src/meta/model/src/actor_dispatcher.rs
index 7d40af6967d35..1ef2e251dd750 100644
--- a/src/meta/model/src/actor_dispatcher.rs
+++ b/src/meta/model/src/actor_dispatcher.rs
@@ -23,7 +23,7 @@ use crate::{ActorId, ActorMapping, FragmentId, I32Array};
 #[derive(
     Hash, Copy, Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize,
 )]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum DispatcherType {
     #[sea_orm(string_value = "HASH")]
     Hash,
diff --git a/src/meta/model/src/catalog_version.rs b/src/meta/model/src/catalog_version.rs
index cfcd90c12f6a0..3aa797800561e 100644
--- a/src/meta/model/src/catalog_version.rs
+++ b/src/meta/model/src/catalog_version.rs
@@ -16,7 +16,7 @@ use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum VersionCategory {
     #[sea_orm(string_value = "NOTIFICATION")]
     Notification,
diff --git a/src/meta/model/src/connection.rs b/src/meta/model/src/connection.rs
index a6cfa4aefb58c..dce0daa462fc5 100644
--- a/src/meta/model/src/connection.rs
+++ b/src/meta/model/src/connection.rs
@@ -26,6 +26,8 @@ pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
     pub connection_id: ConnectionId,
     pub name: String,
+
+    // todo: Private link service has been deprecated, consider using a new field for the connection info
     pub info: PrivateLinkService,
 }
 
diff --git a/src/meta/model/src/fragment.rs b/src/meta/model/src/fragment.rs
index 9a365f37377db..45f59ed3c0aec 100644
--- a/src/meta/model/src/fragment.rs
+++ b/src/meta/model/src/fragment.rs
@@ -33,7 +33,7 @@ pub struct Model {
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum DistributionType {
     #[sea_orm(string_value = "SINGLE")]
     Single,
diff --git a/src/meta/model/src/function.rs b/src/meta/model/src/function.rs
index 5589c1daa023f..0fea52c6c3488 100644
--- a/src/meta/model/src/function.rs
+++ b/src/meta/model/src/function.rs
@@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize};
 use crate::{DataType, DataTypeArray, FunctionId};
 
 #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum FunctionKind {
     #[sea_orm(string_value = "Scalar")]
     Scalar,
diff --git a/src/meta/model/src/hummock_gc_history.rs b/src/meta/model/src/hummock_gc_history.rs
new file mode 100644
index 0000000000000..5a97d79ae5346
--- /dev/null
+++ b/src/meta/model/src/hummock_gc_history.rs
@@ -0,0 +1,31 @@
+// Copyright 2024 RisingWave Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use sea_orm::entity::prelude::*;
+use sea_orm::{DeriveEntityModel, DeriveRelation, EnumIter};
+
+use crate::HummockSstableObjectId;
+
+#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Default)]
+#[sea_orm(table_name = "hummock_gc_history")]
+pub struct Model {
+    #[sea_orm(primary_key, auto_increment = false)]
+    pub object_id: HummockSstableObjectId,
+    pub mark_delete_at: DateTime,
+}
+
+impl ActiveModelBehavior for ActiveModel {}
+
+#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
+pub enum Relation {}
diff --git a/src/meta/model/src/lib.rs b/src/meta/model/src/lib.rs
index e0db07f4709ca..6610484a89185 100644
--- a/src/meta/model/src/lib.rs
+++ b/src/meta/model/src/lib.rs
@@ -36,6 +36,7 @@ pub mod database;
 pub mod fragment;
 pub mod function;
 pub mod hummock_epoch_to_version;
+pub mod hummock_gc_history;
 pub mod hummock_pinned_snapshot;
 pub mod hummock_pinned_version;
 pub mod hummock_sequence;
@@ -92,7 +93,7 @@ pub type FragmentId = i32;
 pub type ActorId = i32;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum JobStatus {
     #[sea_orm(string_value = "INITIAL")]
     Initial,
@@ -124,7 +125,7 @@ impl From<JobStatus> for PbStreamJobState {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum CreateType {
     #[sea_orm(string_value = "BACKGROUND")]
     Background,
diff --git a/src/meta/model/src/object.rs b/src/meta/model/src/object.rs
index 4ee0e5a4f4c49..a9a6fbe524881 100644
--- a/src/meta/model/src/object.rs
+++ b/src/meta/model/src/object.rs
@@ -18,7 +18,7 @@ use serde::{Deserialize, Serialize};
 use crate::{DatabaseId, ObjectId, SchemaId, UserId};
 
 #[derive(Clone, Debug, PartialEq, Eq, Copy, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum ObjectType {
     #[sea_orm(string_value = "DATABASE")]
     Database,
diff --git a/src/meta/model/src/secret.rs b/src/meta/model/src/secret.rs
index 0d122267bb4bc..bea3f87f56bc9 100644
--- a/src/meta/model/src/secret.rs
+++ b/src/meta/model/src/secret.rs
@@ -23,7 +23,6 @@ pub struct Model {
     #[sea_orm(primary_key, auto_increment = false)]
     pub secret_id: i32,
     pub name: String,
-    #[sea_orm(column_type = "Binary(BlobSize::Blob(None))")]
     pub value: Vec<u8>,
 }
 
diff --git a/src/meta/model/src/sink.rs b/src/meta/model/src/sink.rs
index c5a72fbb748c8..39b77fcc6381b 100644
--- a/src/meta/model/src/sink.rs
+++ b/src/meta/model/src/sink.rs
@@ -23,7 +23,7 @@ use crate::{
 };
 
 #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum SinkType {
     #[sea_orm(string_value = "APPEND_ONLY")]
     AppendOnly,
diff --git a/src/meta/model/src/table.rs b/src/meta/model/src/table.rs
index 0a47208ff7351..b8ba38d438b7c 100644
--- a/src/meta/model/src/table.rs
+++ b/src/meta/model/src/table.rs
@@ -29,7 +29,7 @@ use crate::{
 #[derive(
     Clone, Debug, PartialEq, Hash, Copy, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize,
 )]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum TableType {
     #[sea_orm(string_value = "TABLE")]
     Table,
@@ -65,7 +65,7 @@ impl From<PbTableType> for TableType {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum HandleConflictBehavior {
     #[sea_orm(string_value = "OVERWRITE")]
     Overwrite,
@@ -209,7 +209,15 @@ impl From<PbTable> for ActiveModel {
     fn from(pb_table: PbTable) -> Self {
         let table_type = pb_table.table_type();
         let handle_pk_conflict_behavior = pb_table.handle_pk_conflict_behavior();
-        let vnode_count = pb_table.vnode_count();
+
+        // `PbTable` here should be sourced from the wire, not from persistence.
+        // A placeholder `maybe_vnode_count` field should be treated as `NotSet`, instead of calling
+        // the compatibility code.
+        let vnode_count = pb_table
+            .vnode_count_inner()
+            .value_opt()
+            .map(|v| v as _)
+            .map_or(NotSet, Set);
 
         let fragment_id = if pb_table.fragment_id == OBJECT_ID_PLACEHOLDER {
             NotSet
@@ -258,7 +266,7 @@ impl From<PbTable> for ActiveModel {
             retention_seconds: Set(pb_table.retention_seconds.map(|i| i as _)),
             incoming_sinks: Set(pb_table.incoming_sinks.into()),
             cdc_table_id: Set(pb_table.cdc_table_id),
-            vnode_count: Set(vnode_count as _),
+            vnode_count,
         }
     }
 }
diff --git a/src/meta/model/src/user_privilege.rs b/src/meta/model/src/user_privilege.rs
index 48f9a38a5504c..a0743a1f0b56b 100644
--- a/src/meta/model/src/user_privilege.rs
+++ b/src/meta/model/src/user_privilege.rs
@@ -18,8 +18,10 @@ use serde::{Deserialize, Serialize};
 
 use crate::{ObjectId, PrivilegeId, UserId};
 
-#[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[derive(
+    Clone, Copy, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize,
+)]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum Action {
     #[sea_orm(string_value = "INSERT")]
     Insert,
diff --git a/src/meta/model/src/worker.rs b/src/meta/model/src/worker.rs
index ee2ab9b22d3c8..82b13184eabe6 100644
--- a/src/meta/model/src/worker.rs
+++ b/src/meta/model/src/worker.rs
@@ -21,7 +21,7 @@ use serde::{Deserialize, Serialize};
 use crate::{TransactionId, WorkerId};
 
 #[derive(Clone, Debug, Hash, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum WorkerType {
     #[sea_orm(string_value = "FRONTEND")]
     Frontend,
@@ -61,7 +61,7 @@ impl From<WorkerType> for PbWorkerType {
 }
 
 #[derive(Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize, Deserialize)]
-#[sea_orm(rs_type = "String", db_type = "String(None)")]
+#[sea_orm(rs_type = "String", db_type = "string(None)")]
 pub enum WorkerStatus {
     #[sea_orm(string_value = "STARTING")]
     Starting,
diff --git a/src/meta/node/src/lib.rs b/src/meta/node/src/lib.rs
index 3058a04c15f91..0aa4c06f707e0 100644
--- a/src/meta/node/src/lib.rs
+++ b/src/meta/node/src/lib.rs
@@ -231,6 +231,7 @@ pub fn start(
         let listen_addr = opts.listen_addr.parse().unwrap();
         let dashboard_addr = opts.dashboard_host.map(|x| x.parse().unwrap());
         let prometheus_addr = opts.prometheus_listener_addr.map(|x| x.parse().unwrap());
+        let meta_store_config = config.meta.meta_store_config.clone();
         let backend = match config.meta.backend {
             MetaBackend::Mem => MetaStoreBackend::Mem,
             MetaBackend::Sql => MetaStoreBackend::Sql {
@@ -239,6 +240,7 @@ pub fn start(
                     .expect("sql endpoint is required")
                     .expose_secret()
                     .to_string(),
+                config: meta_store_config,
             },
             MetaBackend::Sqlite => MetaStoreBackend::Sql {
                 endpoint: format!(
@@ -247,6 +249,7 @@ pub fn start(
                         .expect("sql endpoint is required")
                         .expose_secret()
                 ),
+                config: meta_store_config,
             },
             MetaBackend::Postgres => MetaStoreBackend::Sql {
                 endpoint: format!(
@@ -258,6 +261,7 @@ pub fn start(
                         .expose_secret(),
                     opts.sql_database
                 ),
+                config: meta_store_config,
             },
             MetaBackend::Mysql => MetaStoreBackend::Sql {
                 endpoint: format!(
@@ -269,6 +273,7 @@ pub fn start(
                         .expose_secret(),
                     opts.sql_database
                 ),
+                config: meta_store_config,
             },
         };
         validate_config(&config);
@@ -367,6 +372,7 @@ pub fn start(
                 min_sst_retention_time_sec: config.meta.min_sst_retention_time_sec,
                 full_gc_interval_sec: config.meta.full_gc_interval_sec,
                 full_gc_object_limit: config.meta.full_gc_object_limit,
+                gc_history_retention_time_sec: config.meta.gc_history_retention_time_sec,
                 max_inflight_time_travel_query: config.meta.max_inflight_time_travel_query,
                 enable_committed_sst_sanity_check: config.meta.enable_committed_sst_sanity_check,
                 periodic_compaction_interval_sec: config.meta.periodic_compaction_interval_sec,
@@ -386,13 +392,18 @@ pub fn start(
                 periodic_tombstone_reclaim_compaction_interval_sec: config
                     .meta
                     .periodic_tombstone_reclaim_compaction_interval_sec,
-                periodic_scheduling_compaction_group_interval_sec: config
+                periodic_scheduling_compaction_group_split_interval_sec: config
                     .meta
-                    .periodic_scheduling_compaction_group_interval_sec,
-                split_group_size_limit: config.meta.split_group_size_limit,
-                min_table_split_size: config.meta.move_table_size_limit,
-                table_write_throughput_threshold: config.meta.table_write_throughput_threshold,
-                min_table_split_write_throughput: config.meta.min_table_split_write_throughput,
+                    .periodic_scheduling_compaction_group_split_interval_sec,
+                periodic_scheduling_compaction_group_merge_interval_sec: config
+                    .meta
+                    .periodic_scheduling_compaction_group_merge_interval_sec,
+                table_high_write_throughput_threshold: config
+                    .meta
+                    .table_high_write_throughput_threshold,
+                table_low_write_throughput_threshold: config
+                    .meta
+                    .table_low_write_throughput_threshold,
                 partition_vnode_count: config.meta.partition_vnode_count,
                 compact_task_table_size_partition_threshold_low: config
                     .meta
@@ -423,6 +434,19 @@ pub fn start(
                     .developer
                     .enable_check_task_level_overlap,
                 enable_dropped_column_reclaim: config.meta.enable_dropped_column_reclaim,
+                split_group_size_ratio: config.meta.split_group_size_ratio,
+                table_stat_high_write_throughput_ratio_for_split: config
+                    .meta
+                    .table_stat_high_write_throughput_ratio_for_split,
+                table_stat_low_write_throughput_ratio_for_merge: config
+                    .meta
+                    .table_stat_low_write_throughput_ratio_for_merge,
+                table_stat_throuput_window_seconds_for_split: config
+                    .meta
+                    .table_stat_throuput_window_seconds_for_split,
+                table_stat_throuput_window_seconds_for_merge: config
+                    .meta
+                    .table_stat_throuput_window_seconds_for_merge,
                 object_store_config: config.storage.object_store,
                 max_trivial_move_task_count_per_loop: config
                     .meta
@@ -431,9 +455,6 @@ pub fn start(
                 max_get_task_probe_times: config.meta.developer.max_get_task_probe_times,
                 secret_store_private_key,
                 temp_secret_file_dir: opts.temp_secret_file_dir,
-                table_info_statistic_history_times: config
-                    .storage
-                    .table_info_statistic_history_times,
                 actor_cnt_per_worker_parallelism_hard_limit: config
                     .meta
                     .developer
diff --git a/src/meta/node/src/server.rs b/src/meta/node/src/server.rs
index b556c4ca34524..2ea5480eba6f6 100644
--- a/src/meta/node/src/server.rs
+++ b/src/meta/node/src/server.rs
@@ -25,6 +25,7 @@ use risingwave_common::telemetry::manager::TelemetryManager;
 use risingwave_common::telemetry::{report_scarf_enabled, report_to_scarf, telemetry_env_enabled};
 use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_common_service::{MetricsManager, TracingExtractLayer};
+use risingwave_meta::barrier::GlobalBarrierManager;
 use risingwave_meta::controller::catalog::CatalogController;
 use risingwave_meta::controller::cluster::ClusterController;
 use risingwave_meta::controller::IN_MEMORY_STORE;
@@ -80,13 +81,12 @@ use thiserror_ext::AsReport;
 use tokio::sync::watch;
 
 use crate::backup_restore::BackupManager;
-use crate::barrier::{BarrierScheduler, GlobalBarrierManager};
+use crate::barrier::BarrierScheduler;
 use crate::controller::system_param::SystemParamsController;
 use crate::controller::SqlMetaStore;
 use crate::hummock::HummockManager;
 use crate::manager::sink_coordination::SinkCoordinatorManager;
 use crate::manager::{IdleManager, MetaOpts, MetaSrvEnv};
-use crate::rpc::cloud_provider::AwsEc2Client;
 use crate::rpc::election::sql::{MySqlDriver, PostgresDriver, SqlBackendElectionClient};
 use crate::rpc::metrics::{
     start_fragment_info_monitor, start_worker_info_monitor, GLOBAL_META_METRICS,
@@ -147,14 +147,15 @@ pub async fn rpc_serve(
             )
             .await
         }
-        MetaStoreBackend::Sql { endpoint } => {
+        MetaStoreBackend::Sql { endpoint, config } => {
             let is_sqlite = DbBackend::Sqlite.is_prefix_of(&endpoint);
             let mut options = sea_orm::ConnectOptions::new(endpoint);
             options
-                .max_connections(10)
-                .connect_timeout(Duration::from_secs(10))
-                .idle_timeout(Duration::from_secs(30))
-                .acquire_timeout(Duration::from_secs(30));
+                .max_connections(config.max_connections)
+                .min_connections(config.min_connections)
+                .connect_timeout(Duration::from_secs(config.connection_timeout_sec))
+                .idle_timeout(Duration::from_secs(config.idle_timeout_sec))
+                .acquire_timeout(Duration::from_secs(config.acquire_timeout_sec));
 
             if is_sqlite {
                 // Since Sqlite is prone to the error "(code: 5) database is locked" under concurrent access,
@@ -438,11 +439,8 @@ pub async fn start_service_as_election_leader(
         None
     };
 
-    let (barrier_scheduler, scheduled_barriers) = BarrierScheduler::new_pair(
-        hummock_manager.clone(),
-        meta_metrics.clone(),
-        system_params_reader.checkpoint_frequency() as usize,
-    );
+    let (barrier_scheduler, scheduled_barriers) =
+        BarrierScheduler::new_pair(hummock_manager.clone(), meta_metrics.clone());
 
     // Initialize services.
     let backup_manager = BackupManager::new(
@@ -489,17 +487,17 @@ pub async fn start_service_as_election_leader(
         env.clone(),
     ));
 
-    let barrier_manager = GlobalBarrierManager::new(
+    let (barrier_manager, join_handle, shutdown_rx) = GlobalBarrierManager::start(
         scheduled_barriers,
         env.clone(),
         metadata_manager.clone(),
         hummock_manager.clone(),
         source_manager.clone(),
         sink_manager.clone(),
-        meta_metrics.clone(),
         scale_controller.clone(),
     )
     .await;
+    sub_tasks.push((join_handle, shutdown_rx));
 
     {
         let source_manager = source_manager.clone();
@@ -531,21 +529,12 @@ pub async fn start_service_as_election_leader(
         compactor_manager.clone(),
     ));
 
-    let mut aws_cli = None;
-    if let Some(my_vpc_id) = &env.opts.vpc_id
-        && let Some(security_group_id) = &env.opts.security_group_id
-    {
-        let cli = AwsEc2Client::new(my_vpc_id, security_group_id).await;
-        aws_cli = Some(cli);
-    }
-
     let ddl_srv = DdlServiceImpl::new(
         env.clone(),
-        aws_cli.clone(),
         metadata_manager.clone(),
         stream_manager.clone(),
         source_manager.clone(),
-        barrier_manager.context().clone(),
+        barrier_manager.clone(),
         sink_manager.clone(),
         meta_metrics.clone(),
     )
@@ -557,12 +546,11 @@ pub async fn start_service_as_election_leader(
         metadata_manager.clone(),
         source_manager,
         stream_manager.clone(),
-        barrier_manager.context().clone(),
+        barrier_manager.clone(),
         scale_controller.clone(),
     );
 
-    let cluster_srv =
-        ClusterServiceImpl::new(metadata_manager.clone(), barrier_manager.context().clone());
+    let cluster_srv = ClusterServiceImpl::new(metadata_manager.clone(), barrier_manager.clone());
     let stream_srv = StreamServiceImpl::new(
         env.clone(),
         barrier_scheduler.clone(),
@@ -586,7 +574,7 @@ pub async fn start_service_as_election_leader(
     let session_params_srv = SessionParamsServiceImpl::new(env.session_params_manager_impl_ref());
     let serving_srv =
         ServingServiceImpl::new(serving_vnode_mapping.clone(), metadata_manager.clone());
-    let cloud_srv = CloudServiceImpl::new(metadata_manager.clone(), aws_cli);
+    let cloud_srv = CloudServiceImpl::new();
     let event_log_srv = EventLogServiceImpl::new(env.event_log_manager_ref());
     let cluster_limit_srv = ClusterLimitServiceImpl::new(env.clone(), metadata_manager.clone());
 
@@ -629,12 +617,11 @@ pub async fn start_service_as_election_leader(
         .await,
     );
 
-    if cfg!(not(test)) {
+    {
         sub_tasks.push(ClusterController::start_heartbeat_checker(
             metadata_manager.cluster_controller.clone(),
             Duration::from_secs(1),
         ));
-        sub_tasks.push(GlobalBarrierManager::start(barrier_manager));
 
         if !env.opts.disable_automatic_parallelism_control {
             sub_tasks.push(stream_manager.start_auto_parallelism_monitor());
@@ -705,6 +692,7 @@ pub async fn start_service_as_election_leader(
         .add_service(MetaMemberServiceServer::new(meta_member_srv))
         .add_service(DdlServiceServer::new(ddl_srv).max_decoding_message_size(usize::MAX))
         .add_service(UserServiceServer::new(user_srv))
+        .add_service(CloudServiceServer::new(cloud_srv))
         .add_service(ScaleServiceServer::new(scale_srv).max_decoding_message_size(usize::MAX))
         .add_service(HealthServer::new(health_srv))
         .add_service(BackupServiceServer::new(backup_srv))
@@ -712,7 +700,6 @@ pub async fn start_service_as_election_leader(
         .add_service(SessionParamServiceServer::new(session_params_srv))
         .add_service(TelemetryInfoServiceServer::new(telemetry_srv))
         .add_service(ServingServiceServer::new(serving_srv))
-        .add_service(CloudServiceServer::new(cloud_srv))
         .add_service(SinkCoordinationServiceServer::new(sink_coordination_srv))
         .add_service(EventLogServiceServer::new(event_log_srv))
         .add_service(ClusterLimitServiceServer::new(cluster_limit_srv));
diff --git a/src/meta/service/src/cloud_service.rs b/src/meta/service/src/cloud_service.rs
index e913b91826b6f..553a8189116c3 100644
--- a/src/meta/service/src/cloud_service.rs
+++ b/src/meta/service/src/cloud_service.rs
@@ -18,14 +18,10 @@ use std::sync::LazyLock;
 use async_trait::async_trait;
 use regex::Regex;
 use risingwave_connector::error::ConnectorResult;
-use risingwave_connector::source::kafka::private_link::insert_privatelink_broker_rewrite_map;
 use risingwave_connector::source::{
     ConnectorProperties, SourceEnumeratorContext, SourceProperties, SplitEnumerator,
 };
 use risingwave_connector::{dispatch_source_prop, WithOptionsSecResolved};
-use risingwave_meta::manager::MetadataManager;
-use risingwave_meta_model::ConnectionId;
-use risingwave_pb::catalog::connection::Info::PrivateLinkService;
 use risingwave_pb::cloud_service::cloud_service_server::CloudService;
 use risingwave_pb::cloud_service::rw_cloud_validate_source_response::{Error, ErrorType};
 use risingwave_pb::cloud_service::{
@@ -33,20 +29,11 @@ use risingwave_pb::cloud_service::{
 };
 use thiserror_ext::AsReport;
 use tonic::{Request, Response, Status};
-
-use crate::rpc::cloud_provider::AwsEc2Client;
-
-pub struct CloudServiceImpl {
-    metadata_manager: MetadataManager,
-    aws_client: Option<AwsEc2Client>,
-}
+pub struct CloudServiceImpl {}
 
 impl CloudServiceImpl {
-    pub fn new(metadata_manager: MetadataManager, aws_client: Option<AwsEc2Client>) -> Self {
-        Self {
-            metadata_manager,
-            aws_client,
-        }
+    pub fn new() -> Self {
+        Self {}
     }
 }
 
@@ -76,70 +63,7 @@ impl CloudService for CloudServiceImpl {
                 "unexpected source type, only kafka source is supported",
             ));
         }
-        let mut source_cfg: BTreeMap<String, String> = req.source_config.into_iter().collect();
-        // if connection_id provided, check whether endpoint service is available and resolve
-        // broker rewrite map currently only support aws privatelink connection
-        if let Some(connection_id_str) = source_cfg.get("connection.id") {
-            let connection_id = connection_id_str.parse::<ConnectionId>().map_err(|e| {
-                Status::invalid_argument(format!(
-                    "connection.id is not an integer: {}",
-                    e.as_report()
-                ))
-            })?;
-
-            let connection = self
-                .metadata_manager
-                .catalog_controller
-                .get_connection_by_id(connection_id)
-                .await;
-
-            if let Err(e) = connection {
-                return Ok(new_rwc_validate_fail_response(
-                    ErrorType::PrivatelinkConnectionNotFound,
-                    e.to_report_string(),
-                ));
-            }
-            if let Some(PrivateLinkService(service)) = connection.unwrap().info {
-                if self.aws_client.is_none() {
-                    return Ok(new_rwc_validate_fail_response(
-                        ErrorType::AwsClientNotConfigured,
-                        "AWS client is not configured".to_string(),
-                    ));
-                }
-                let cli = self.aws_client.as_ref().unwrap();
-                let privatelink_status = cli
-                    .is_vpc_endpoint_ready(service.endpoint_id.as_str())
-                    .await;
-                match privatelink_status {
-                    Err(e) => {
-                        return Ok(new_rwc_validate_fail_response(
-                            ErrorType::PrivatelinkUnavailable,
-                            e.to_report_string(),
-                        ));
-                    }
-                    Ok(false) => {
-                        return Ok(new_rwc_validate_fail_response(
-                            ErrorType::PrivatelinkUnavailable,
-                            format!("Private link endpoint {} is not ready", service.endpoint_id,),
-                        ));
-                    }
-                    _ => (),
-                };
-                if let Err(e) =
-                    insert_privatelink_broker_rewrite_map(&mut source_cfg, Some(&service), None)
-                {
-                    return Ok(new_rwc_validate_fail_response(
-                        ErrorType::PrivatelinkResolveErr,
-                        e.to_report_string(),
-                    ));
-                }
-            } else {
-                return Ok(new_rwc_validate_fail_response(
-                    ErrorType::PrivatelinkResolveErr,
-                    format!("connection {} has no info available", connection_id),
-                ));
-            }
-        }
+        let source_cfg: BTreeMap<String, String> = req.source_config.into_iter().collect();
 
         // XXX: We can't use secret in cloud validate source.
         let source_cfg = WithOptionsSecResolved::without_secrets(source_cfg);
diff --git a/src/meta/service/src/ddl_service.rs b/src/meta/service/src/ddl_service.rs
index 1578813e2ead9..e59c4a4100141 100644
--- a/src/meta/service/src/ddl_service.rs
+++ b/src/meta/service/src/ddl_service.rs
@@ -23,14 +23,8 @@ use risingwave_common::types::DataType;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_connector::sink::catalog::SinkId;
 use risingwave_meta::manager::{EventLogManagerRef, MetadataManager};
-use risingwave_meta::rpc::ddl_controller::fill_table_stream_graph_info;
 use risingwave_meta::rpc::metrics::MetaMetrics;
-use risingwave_pb::catalog::connection::private_link_service::{
-    PbPrivateLinkProvider, PrivateLinkProvider,
-};
-use risingwave_pb::catalog::connection::PbPrivateLinkService;
-use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
-use risingwave_pb::catalog::{connection, Comment, Connection, CreateType, Secret, Table};
+use risingwave_pb::catalog::{Comment, CreateType, Secret, Table};
 use risingwave_pb::common::worker_node::State;
 use risingwave_pb::common::WorkerType;
 use risingwave_pb::ddl_service::ddl_service_server::DdlService;
@@ -44,12 +38,11 @@ use tonic::{Request, Response, Status};
 use crate::barrier::BarrierManagerRef;
 use crate::manager::sink_coordination::SinkCoordinatorManager;
 use crate::manager::{MetaSrvEnv, StreamingJob};
-use crate::rpc::cloud_provider::AwsEc2Client;
 use crate::rpc::ddl_controller::{
     DdlCommand, DdlController, DropMode, ReplaceTableInfo, StreamingJobId,
 };
 use crate::stream::{GlobalStreamManagerRef, SourceManagerRef};
-use crate::{MetaError, MetaResult};
+use crate::MetaError;
 
 #[derive(Clone)]
 pub struct DdlServiceImpl {
@@ -58,7 +51,6 @@ pub struct DdlServiceImpl {
     metadata_manager: MetadataManager,
     sink_manager: SinkCoordinatorManager,
     ddl_controller: DdlController,
-    aws_client: Arc<Option<AwsEc2Client>>,
     meta_metrics: Arc<MetaMetrics>,
 }
 
@@ -66,7 +58,6 @@ impl DdlServiceImpl {
     #[allow(clippy::too_many_arguments)]
     pub async fn new(
         env: MetaSrvEnv,
-        aws_client: Option<AwsEc2Client>,
         metadata_manager: MetadataManager,
         stream_manager: GlobalStreamManagerRef,
         source_manager: SourceManagerRef,
@@ -74,47 +65,45 @@ impl DdlServiceImpl {
         sink_manager: SinkCoordinatorManager,
         meta_metrics: Arc<MetaMetrics>,
     ) -> Self {
-        let aws_cli_ref = Arc::new(aws_client);
         let ddl_controller = DdlController::new(
             env.clone(),
             metadata_manager.clone(),
             stream_manager,
             source_manager,
             barrier_manager,
-            aws_cli_ref.clone(),
         )
         .await;
         Self {
             env,
             metadata_manager,
-            ddl_controller,
-            aws_client: aws_cli_ref,
             sink_manager,
+            ddl_controller,
             meta_metrics,
         }
     }
 
-    fn extract_replace_table_info(change: ReplaceTablePlan) -> ReplaceTableInfo {
-        let job_type = change.get_job_type().unwrap_or_default();
-        let mut source = change.source;
-        let mut fragment_graph = change.fragment_graph.unwrap();
-        let mut table = change.table.unwrap();
-        if let Some(OptionalAssociatedSourceId::AssociatedSourceId(source_id)) =
-            table.optional_associated_source_id
-        {
-            source.as_mut().unwrap().id = source_id;
-            fill_table_stream_graph_info(&mut source, &mut table, job_type, &mut fragment_graph);
-        }
-        let table_col_index_mapping = change
-            .table_col_index_mapping
+    fn extract_replace_table_info(
+        ReplaceTablePlan {
+            table,
+            fragment_graph,
+            table_col_index_mapping,
+            source,
+            job_type,
+        }: ReplaceTablePlan,
+    ) -> ReplaceTableInfo {
+        let table = table.unwrap();
+        let col_index_mapping = table_col_index_mapping
             .as_ref()
             .map(ColIndexMapping::from_protobuf);
 
-        let stream_job = StreamingJob::Table(source, table, job_type);
         ReplaceTableInfo {
-            streaming_job: stream_job,
-            fragment_graph,
-            col_index_mapping: table_col_index_mapping,
+            streaming_job: StreamingJob::Table(
+                source,
+                table,
+                TableJobType::try_from(job_type).unwrap(),
+            ),
+            fragment_graph: fragment_graph.unwrap(),
+            col_index_mapping,
         }
     }
 }
@@ -231,11 +220,6 @@ impl DdlService for DdlServiceImpl {
         let req = request.into_inner();
         let source = req.get_source()?.clone();
 
-        // validate connection before starting the DDL procedure
-        if let Some(connection_id) = source.connection_id {
-            self.validate_connection(connection_id).await?;
-        }
-
         match req.fragment_graph {
             None => {
                 let version = self
@@ -297,11 +281,6 @@ impl DdlService for DdlServiceImpl {
         let fragment_graph = req.get_fragment_graph()?.clone();
         let affected_table_change = req.get_affected_table_change().cloned().ok();
 
-        // validate connection before starting the DDL procedure
-        if let Some(connection_id) = sink.connection_id {
-            self.validate_connection(connection_id).await?;
-        }
-
         let stream_job = match &affected_table_change {
             None => StreamingJob::Sink(sink, None),
             Some(change) => {
@@ -748,63 +727,10 @@ impl DdlService for DdlServiceImpl {
         }
 
         match req.payload.unwrap() {
-            create_connection_request::Payload::PrivateLink(link) => {
-                // currently we only support AWS
-                let private_link_svc = match link.get_provider()? {
-                    PbPrivateLinkProvider::Mock => PbPrivateLinkService {
-                        provider: link.provider,
-                        service_name: String::new(),
-                        endpoint_id: String::new(),
-                        endpoint_dns_name: String::new(),
-                        dns_entries: HashMap::new(),
-                    },
-                    PbPrivateLinkProvider::Aws => {
-                        if let Some(aws_cli) = self.aws_client.as_ref() {
-                            let tags_env = self
-                                .env
-                                .opts
-                                .privatelink_endpoint_default_tags
-                                .as_ref()
-                                .map(|tags| {
-                                    tags.iter()
-                                        .map(|(key, val)| (key.as_str(), val.as_str()))
-                                        .collect()
-                                });
-                            aws_cli
-                                .create_aws_private_link(
-                                    &link.service_name,
-                                    link.tags.as_deref(),
-                                    tags_env,
-                                )
-                                .await?
-                        } else {
-                            return Err(Status::from(MetaError::unavailable(
-                                "AWS client is not configured",
-                            )));
-                        }
-                    }
-                    PbPrivateLinkProvider::Unspecified => {
-                        return Err(Status::invalid_argument("Privatelink provider unspecified"));
-                    }
-                };
-                let connection = Connection {
-                    id: 0,
-                    schema_id: req.schema_id,
-                    database_id: req.database_id,
-                    name: req.name,
-                    owner: req.owner_id,
-                    info: Some(connection::Info::PrivateLinkService(private_link_svc)),
-                };
-
-                // save private link info to catalog
-                let version = self
-                    .ddl_controller
-                    .run_command(DdlCommand::CreateConnection(connection))
-                    .await?;
-
-                Ok(Response::new(CreateConnectionResponse { version }))
+            create_connection_request::Payload::PrivateLink(_) => {
+                panic!("Private Link Connection has been deprecated")
             }
-        }
+        };
     }
 
     async fn list_connections(
@@ -1086,32 +1012,22 @@ impl DdlService for DdlServiceImpl {
 
         Ok(Response::new(AutoSchemaChangeResponse {}))
     }
-}
 
-impl DdlServiceImpl {
-    async fn validate_connection(&self, connection_id: u32) -> MetaResult<()> {
-        let connection = self
-            .metadata_manager
-            .catalog_controller
-            .get_connection_by_id(connection_id as _)
+    async fn alter_swap_rename(
+        &self,
+        request: Request<AlterSwapRenameRequest>,
+    ) -> Result<Response<AlterSwapRenameResponse>, Status> {
+        let req = request.into_inner();
+
+        let version = self
+            .ddl_controller
+            .run_command(DdlCommand::AlterSwapRename(req.object.unwrap()))
             .await?;
-        if let Some(connection::Info::PrivateLinkService(svc)) = &connection.info {
-            // skip all checks for mock connection
-            if svc.get_provider()? == PrivateLinkProvider::Mock {
-                return Ok(());
-            }
 
-            // check whether private link is ready
-            if let Some(aws_cli) = self.aws_client.as_ref() {
-                if !aws_cli.is_vpc_endpoint_ready(&svc.endpoint_id).await? {
-                    return Err(MetaError::from(anyhow!(
-                        "Private link endpoint {} is not ready",
-                        svc.endpoint_id
-                    )));
-                }
-            }
-        }
-        Ok(())
+        Ok(Response::new(AlterSwapRenameResponse {
+            status: None,
+            version,
+        }))
     }
 }
 
diff --git a/src/meta/service/src/event_log_service.rs b/src/meta/service/src/event_log_service.rs
index c7b148e28a58f..a118d6e9ca267 100644
--- a/src/meta/service/src/event_log_service.rs
+++ b/src/meta/service/src/event_log_service.rs
@@ -50,6 +50,9 @@ impl EventLogService for EventLogServiceImpl {
             risingwave_pb::meta::add_event_log_request::Event::WorkerNodePanic(e) => {
                 risingwave_pb::meta::event_log::Event::WorkerNodePanic(e)
             }
+            risingwave_pb::meta::add_event_log_request::Event::SinkFail(e) => {
+                risingwave_pb::meta::event_log::Event::SinkFail(e)
+            }
         };
         self.event_log_manager.add_event_logs(vec![e]);
         Ok(Response::new(AddEventLogResponse {}))
diff --git a/src/meta/service/src/hummock_service.rs b/src/meta/service/src/hummock_service.rs
index c3c3fcd6aa0f0..38562ab53ba26 100644
--- a/src/meta/service/src/hummock_service.rs
+++ b/src/meta/service/src/hummock_service.rs
@@ -257,11 +257,16 @@ impl HummockManagerService for HummockServiceImpl {
             req.total_object_count,
             req.total_object_size,
         );
+        let pinned_by_metadata_backup = self.vacuum_manager.backup_manager.list_pinned_ssts();
         // The following operation takes some time, so we do it in dedicated task and responds the
         // RPC immediately.
         tokio::spawn(async move {
             match hummock_manager
-                .complete_full_gc(req.object_ids, req.next_start_after)
+                .complete_full_gc(
+                    req.object_ids,
+                    req.next_start_after,
+                    pinned_by_metadata_backup,
+                )
                 .await
             {
                 Ok(number) => {
@@ -382,7 +387,11 @@ impl HummockManagerService for HummockServiceImpl {
             .move_state_tables_to_dedicated_compaction_group(
                 req.group_id,
                 &req.table_ids,
-                req.partition_vnode_count,
+                if req.partition_vnode_count > 0 {
+                    Some(req.partition_vnode_count)
+                } else {
+                    None
+                },
             )
             .await?
             .0;
@@ -534,14 +543,13 @@ impl HummockManagerService for HummockServiceImpl {
             periodic_space_reclaim_compaction_interval_sec,
             periodic_ttl_reclaim_compaction_interval_sec,
             periodic_tombstone_reclaim_compaction_interval_sec,
-            periodic_scheduling_compaction_group_interval_sec,
-            split_group_size_limit,
-            min_table_split_size,
+            periodic_scheduling_compaction_group_split_interval_sec,
             do_not_config_object_storage_lifecycle,
             partition_vnode_count,
-            table_write_throughput_threshold,
-            min_table_split_write_throughput,
-            compaction_task_max_heartbeat_interval_secs
+            table_high_write_throughput_threshold,
+            table_low_write_throughput_threshold,
+            compaction_task_max_heartbeat_interval_secs,
+            periodic_scheduling_compaction_group_merge_interval_sec
         );
         Ok(Response::new(ListHummockMetaConfigResponse { configs }))
     }
diff --git a/src/meta/service/src/scale_service.rs b/src/meta/service/src/scale_service.rs
index 889df5b33904c..11a59aeb93e4f 100644
--- a/src/meta/service/src/scale_service.rs
+++ b/src/meta/service/src/scale_service.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use risingwave_common::catalog::TableId;
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_meta::manager::MetadataManager;
 use risingwave_meta::model::TableParallelism;
 use risingwave_meta::stream::{RescheduleOptions, ScaleControllerRef, WorkerReschedule};
@@ -118,47 +118,54 @@ impl ScaleService for ScaleServiceImpl {
         } = request.into_inner();
 
         let _reschedule_job_lock = self.stream_manager.reschedule_lock_write_guard().await;
-
-        let streaming_job_ids = self
+        for (database_id, worker_reschedules) in self
             .metadata_manager
-            .catalog_controller
-            .get_fragment_job_id(
-                worker_reschedules
-                    .keys()
-                    .map(|id| *id as FragmentId)
-                    .collect(),
-            )
-            .await?;
-
-        let table_parallelisms = streaming_job_ids
-            .into_iter()
-            .map(|id| (TableId::new(id as _), TableParallelism::Custom))
-            .collect();
-
-        self.stream_manager
-            .reschedule_actors(
-                worker_reschedules
-                    .into_iter()
-                    .map(|(fragment_id, reschedule)| {
-                        let PbWorkerReschedule { worker_actor_diff } = reschedule;
-                        (
-                            fragment_id,
-                            WorkerReschedule {
-                                worker_actor_diff: worker_actor_diff
-                                    .into_iter()
-                                    .map(|(worker_id, diff)| (worker_id as _, diff as _))
-                                    .collect(),
-                            },
-                        )
-                    })
-                    .collect(),
-                RescheduleOptions {
-                    resolve_no_shuffle_upstream,
-                    skip_create_new_actors: false,
-                },
-                Some(table_parallelisms),
-            )
-            .await?;
+            .split_fragment_map_by_database(worker_reschedules)
+            .await?
+        {
+            let database_id = DatabaseId::new(database_id as _);
+            let streaming_job_ids = self
+                .metadata_manager
+                .catalog_controller
+                .get_fragment_job_id(
+                    worker_reschedules
+                        .keys()
+                        .map(|id| *id as FragmentId)
+                        .collect(),
+                )
+                .await?;
+
+            let table_parallelisms = streaming_job_ids
+                .into_iter()
+                .map(|id| (TableId::new(id as _), TableParallelism::Custom))
+                .collect();
+
+            self.stream_manager
+                .reschedule_actors(
+                    database_id,
+                    worker_reschedules
+                        .into_iter()
+                        .map(|(fragment_id, reschedule)| {
+                            let PbWorkerReschedule { worker_actor_diff } = reschedule;
+                            (
+                                fragment_id,
+                                WorkerReschedule {
+                                    worker_actor_diff: worker_actor_diff
+                                        .into_iter()
+                                        .map(|(worker_id, diff)| (worker_id as _, diff as _))
+                                        .collect(),
+                                },
+                            )
+                        })
+                        .collect(),
+                    RescheduleOptions {
+                        resolve_no_shuffle_upstream,
+                        skip_create_new_actors: false,
+                    },
+                    Some(table_parallelisms),
+                )
+                .await?;
+        }
 
         Ok(Response::new(RescheduleResponse {
             success: true,
diff --git a/src/meta/service/src/stream_service.rs b/src/meta/service/src/stream_service.rs
index 19fafb5de74f4..1833c19aeaecd 100644
--- a/src/meta/service/src/stream_service.rs
+++ b/src/meta/service/src/stream_service.rs
@@ -15,13 +15,14 @@
 use std::collections::{HashMap, HashSet};
 
 use itertools::Itertools;
-use risingwave_common::catalog::TableId;
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_connector::source::SplitMetaData;
+use risingwave_meta::controller::fragment::StreamingJobInfo;
 use risingwave_meta::manager::{LocalNotification, MetadataManager};
 use risingwave_meta::model;
 use risingwave_meta::model::ActorId;
 use risingwave_meta::stream::{SourceManagerRunningInfo, ThrottleConfig};
-use risingwave_meta_model::{SourceId, StreamingParallelism};
+use risingwave_meta_model::{ObjectId, SourceId, StreamingParallelism};
 use risingwave_pb::meta::cancel_creating_jobs_request::Jobs;
 use risingwave_pb::meta::list_actor_splits_response::FragmentType;
 use risingwave_pb::meta::list_table_fragments_response::{
@@ -71,7 +72,7 @@ impl StreamManagerService for StreamServiceImpl {
         self.env.idle_manager().record_activity();
         let req = request.into_inner();
 
-        let version_id = self.barrier_scheduler.flush(req.checkpoint).await?;
+        let version_id = self.barrier_scheduler.flush(req.database_id.into()).await?;
         Ok(Response::new(FlushResponse {
             status: None,
             hummock_version_id: version_id.to_u64(),
@@ -80,17 +81,27 @@ impl StreamManagerService for StreamServiceImpl {
 
     #[cfg_attr(coverage, coverage(off))]
     async fn pause(&self, _: Request<PauseRequest>) -> Result<Response<PauseResponse>, Status> {
-        self.barrier_scheduler
-            .run_command(Command::pause(PausedReason::Manual))
-            .await?;
+        for database_id in self.metadata_manager.list_active_database_ids().await? {
+            self.barrier_scheduler
+                .run_command(
+                    DatabaseId::new(database_id as _),
+                    Command::pause(PausedReason::Manual),
+                )
+                .await?;
+        }
         Ok(Response::new(PauseResponse {}))
     }
 
     #[cfg_attr(coverage, coverage(off))]
     async fn resume(&self, _: Request<ResumeRequest>) -> Result<Response<ResumeResponse>, Status> {
-        self.barrier_scheduler
-            .run_command(Command::resume(PausedReason::Manual))
-            .await?;
+        for database_id in self.metadata_manager.list_active_database_ids().await? {
+            self.barrier_scheduler
+                .run_command(
+                    DatabaseId::new(database_id as _),
+                    Command::resume(PausedReason::Manual),
+                )
+                .await?;
+        }
         Ok(Response::new(ResumeResponse {}))
     }
 
@@ -122,6 +133,12 @@ impl StreamManagerService for StreamServiceImpl {
             }
         };
 
+        let database_id = self
+            .metadata_manager
+            .catalog_controller
+            .get_object_database_id(request.id as ObjectId)
+            .await?;
+        let database_id = DatabaseId::new(database_id as _);
         // TODO: check whether shared source is correct
         let mutation: ThrottleConfig = actor_to_apply
             .iter()
@@ -137,7 +154,7 @@ impl StreamManagerService for StreamServiceImpl {
             .collect();
         let _i = self
             .barrier_scheduler
-            .run_command(Command::Throttle(mutation))
+            .run_command(database_id, Command::Throttle(mutation))
             .await?;
 
         Ok(Response::new(ApplyThrottleResponse { status: None }))
@@ -222,27 +239,35 @@ impl StreamManagerService for StreamServiceImpl {
         &self,
         _request: Request<ListTableFragmentStatesRequest>,
     ) -> Result<Response<ListTableFragmentStatesResponse>, Status> {
-        let job_states = self
+        let job_infos = self
             .metadata_manager
             .catalog_controller
-            .list_streaming_job_states()
+            .list_streaming_job_infos()
             .await?;
-        let states = job_states
+        let states = job_infos
             .into_iter()
-            .map(|(table_id, state, parallelism, max_parallelism)| {
-                let parallelism = match parallelism {
-                    StreamingParallelism::Adaptive => model::TableParallelism::Adaptive,
-                    StreamingParallelism::Custom => model::TableParallelism::Custom,
-                    StreamingParallelism::Fixed(n) => model::TableParallelism::Fixed(n as _),
-                };
-
-                list_table_fragment_states_response::TableFragmentState {
-                    table_id: table_id as _,
-                    state: PbState::from(state) as _,
-                    parallelism: Some(parallelism.into()),
-                    max_parallelism: max_parallelism as _,
-                }
-            })
+            .map(
+                |StreamingJobInfo {
+                     job_id,
+                     job_status,
+                     parallelism,
+                     max_parallelism,
+                     ..
+                 }| {
+                    let parallelism = match parallelism {
+                        StreamingParallelism::Adaptive => model::TableParallelism::Adaptive,
+                        StreamingParallelism::Custom => model::TableParallelism::Custom,
+                        StreamingParallelism::Fixed(n) => model::TableParallelism::Fixed(n as _),
+                    };
+
+                    list_table_fragment_states_response::TableFragmentState {
+                        table_id: job_id as _,
+                        state: PbState::from(job_status) as _,
+                        parallelism: Some(parallelism.into()),
+                        max_parallelism: max_parallelism as _,
+                    }
+                },
+            )
             .collect_vec();
 
         Ok(Response::new(ListTableFragmentStatesResponse { states }))
@@ -312,7 +337,7 @@ impl StreamManagerService for StreamServiceImpl {
         let dependencies = self
             .metadata_manager
             .catalog_controller
-            .list_object_dependencies()
+            .list_created_object_dependencies()
             .await?;
 
         Ok(Response::new(ListObjectDependenciesResponse {
diff --git a/src/meta/src/backup_restore/utils.rs b/src/meta/src/backup_restore/utils.rs
index fd7039af95fca..0946ce74e5fbf 100644
--- a/src/meta/src/backup_restore/utils.rs
+++ b/src/meta/src/backup_restore/utils.rs
@@ -17,7 +17,7 @@ use std::time::Duration;
 
 use risingwave_backup::error::{BackupError, BackupResult};
 use risingwave_backup::storage::{MetaSnapshotStorageRef, ObjectStoreMetaSnapshotStorage};
-use risingwave_common::config::{MetaBackend, ObjectStoreConfig};
+use risingwave_common::config::{MetaBackend, MetaStoreConfig, ObjectStoreConfig};
 use risingwave_object_store::object::build_remote_object_store;
 use risingwave_object_store::object::object_metrics::ObjectStoreMetrics;
 use sea_orm::DbBackend;
@@ -32,21 +32,25 @@ pub async fn get_meta_store(opts: RestoreOpts) -> BackupResult<SqlMetaStore> {
         MetaBackend::Mem => MetaStoreBackend::Mem,
         MetaBackend::Sql => MetaStoreBackend::Sql {
             endpoint: opts.sql_endpoint,
+            config: MetaStoreConfig::default(),
         },
         MetaBackend::Sqlite => MetaStoreBackend::Sql {
             endpoint: format!("sqlite://{}?mode=rwc", opts.sql_endpoint),
+            config: MetaStoreConfig::default(),
         },
         MetaBackend::Postgres => MetaStoreBackend::Sql {
             endpoint: format!(
                 "postgres://{}:{}@{}/{}",
                 opts.sql_username, opts.sql_password, opts.sql_endpoint, opts.sql_database
             ),
+            config: MetaStoreConfig::default(),
         },
         MetaBackend::Mysql => MetaStoreBackend::Sql {
             endpoint: format!(
                 "mysql://{}:{}@{}/{}",
                 opts.sql_username, opts.sql_password, opts.sql_endpoint, opts.sql_database
             ),
+            config: MetaStoreConfig::default(),
         },
     };
     match meta_store_backend {
@@ -54,19 +58,21 @@ pub async fn get_meta_store(opts: RestoreOpts) -> BackupResult<SqlMetaStore> {
             let conn = sea_orm::Database::connect(IN_MEMORY_STORE).await.unwrap();
             Ok(SqlMetaStore::new(conn))
         }
-        MetaStoreBackend::Sql { endpoint } => {
+        MetaStoreBackend::Sql { endpoint, config } => {
             let max_connection = if DbBackend::Sqlite.is_prefix_of(&endpoint) {
-                // Due to the fact that Sqlite is prone to the error "(code: 5) database is locked" under concurrent access,
+                // Since Sqlite is prone to the error "(code: 5) database is locked" under concurrent access,
                 // here we forcibly specify the number of connections as 1.
                 1
             } else {
-                10
+                config.max_connections
             };
             let mut options = sea_orm::ConnectOptions::new(endpoint);
             options
                 .max_connections(max_connection)
-                .connect_timeout(Duration::from_secs(10))
-                .idle_timeout(Duration::from_secs(30));
+                .min_connections(config.min_connections)
+                .connect_timeout(Duration::from_secs(config.connection_timeout_sec))
+                .idle_timeout(Duration::from_secs(config.idle_timeout_sec))
+                .acquire_timeout(Duration::from_secs(config.acquire_timeout_sec));
             let conn = sea_orm::Database::connect(options)
                 .await
                 .map_err(|e| BackupError::MetaStorage(e.into()))?;
diff --git a/src/meta/src/barrier/command.rs b/src/meta/src/barrier/command.rs
index a5b7efe63098d..86c6a6931ecb8 100644
--- a/src/meta/src/barrier/command.rs
+++ b/src/meta/src/barrier/command.rs
@@ -40,9 +40,9 @@ use risingwave_pb::stream_plan::{
 use risingwave_pb::stream_service::WaitEpochCommitRequest;
 use tracing::warn;
 
-use super::info::{CommandFragmentChanges, InflightGraphInfo};
-use super::trace::TracedEpoch;
-use crate::barrier::{GlobalBarrierManagerContext, InflightSubscriptionInfo};
+use super::info::{CommandFragmentChanges, InflightStreamingJobInfo};
+use crate::barrier::info::BarrierInfo;
+use crate::barrier::{GlobalBarrierWorkerContextImpl, InflightSubscriptionInfo};
 use crate::controller::fragment::InflightFragmentInfo;
 use crate::manager::{DdlType, StreamingJob};
 use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments, TableParallelism};
@@ -78,10 +78,6 @@ pub struct Reschedule {
     /// `Source` and `SourceBackfill` are handled together here.
     pub actor_splits: HashMap<ActorId, Vec<SplitImpl>>,
 
-    /// Whether this fragment is injectable. The injectable means whether the fragment contains
-    /// any executors that are able to receive barrier.
-    pub injectable: bool,
-
     pub newly_created_actors: Vec<(StreamActor, PbActorStatus)>,
 }
 
@@ -109,28 +105,31 @@ impl ReplaceTablePlan {
     fn fragment_changes(&self) -> HashMap<FragmentId, CommandFragmentChanges> {
         let mut fragment_changes = HashMap::new();
         for fragment in self.new_table_fragments.fragments.values() {
-            let fragment_change = CommandFragmentChanges::NewFragment(InflightFragmentInfo {
-                actors: fragment
-                    .actors
-                    .iter()
-                    .map(|actor| {
-                        (
-                            actor.actor_id as i32,
-                            self.new_table_fragments
-                                .actor_status
-                                .get(&actor.actor_id)
-                                .expect("should exist")
-                                .worker_id(),
-                        )
-                    })
-                    .collect(),
-                state_table_ids: fragment
-                    .state_table_ids
-                    .iter()
-                    .map(|table_id| *table_id as ObjectId)
-                    .collect(),
-                is_injectable: TableFragments::is_injectable(fragment.fragment_type_mask),
-            });
+            let fragment_change = CommandFragmentChanges::NewFragment(
+                self.streaming_job.database_id().into(),
+                self.streaming_job.id().into(),
+                InflightFragmentInfo {
+                    actors: fragment
+                        .actors
+                        .iter()
+                        .map(|actor| {
+                            (
+                                actor.actor_id as i32,
+                                self.new_table_fragments
+                                    .actor_status
+                                    .get(&actor.actor_id)
+                                    .expect("should exist")
+                                    .worker_id(),
+                            )
+                        })
+                        .collect(),
+                    state_table_ids: fragment
+                        .state_table_ids
+                        .iter()
+                        .map(|table_id| *table_id as ObjectId)
+                        .collect(),
+                },
+            );
             assert!(fragment_changes
                 .insert(fragment.fragment_id, fragment_change)
                 .is_none());
@@ -187,7 +186,6 @@ impl CreateStreamingJobCommandInfo {
                         .iter()
                         .map(|table_id| *table_id as ObjectId)
                         .collect(),
-                    is_injectable: TableFragments::is_injectable(fragment.fragment_type_mask),
                 },
             )
         })
@@ -206,16 +204,14 @@ pub enum CreateStreamingJobType {
     SnapshotBackfill(SnapshotBackfillInfo),
 }
 
-/// [`Command`] is the input of [`crate::barrier::GlobalBarrierManager`]. For different commands,
+/// [`Command`] is the input of [`crate::barrier::GlobalBarrierWorker`]. For different commands,
 /// it will build different barriers to send, and may do different stuffs after the barrier is
 /// collected.
 #[derive(Debug, Clone, strum::Display)]
 pub enum Command {
-    /// `Plain` command generates a barrier with the mutation it carries.
-    ///
-    /// Barriers from all actors marked as `Created` state will be collected.
-    /// After the barrier is collected, it does nothing.
-    Plain(Option<Mutation>),
+    /// `Flush` command will generate a checkpoint barrier. After the barrier is collected and committed
+    /// all messages before the checkpoint barrier should have been committed.
+    Flush,
 
     /// `Pause` command generates a `Pause` barrier with the provided [`PausedReason`] **only if**
     /// the cluster is not already paused. Otherwise, a barrier with no mutation will be generated.
@@ -252,7 +248,9 @@ pub enum Command {
         info: CreateStreamingJobCommandInfo,
         job_type: CreateStreamingJobType,
     },
-    MergeSnapshotBackfillStreamingJobs(HashMap<TableId, (SnapshotBackfillInfo, InflightGraphInfo)>),
+    MergeSnapshotBackfillStreamingJobs(
+        HashMap<TableId, (SnapshotBackfillInfo, InflightStreamingJobInfo)>,
+    ),
     /// `CancelStreamingJob` command generates a `Stop` barrier including the actors of the given
     /// table fragment.
     ///
@@ -305,10 +303,6 @@ pub enum Command {
 }
 
 impl Command {
-    pub fn barrier() -> Self {
-        Self::Plain(None)
-    }
-
     pub fn pause(reason: PausedReason) -> Self {
         Self::Pause(reason)
     }
@@ -319,7 +313,7 @@ impl Command {
 
     pub(crate) fn fragment_changes(&self) -> Option<HashMap<FragmentId, CommandFragmentChanges>> {
         match self {
-            Command::Plain(_) => None,
+            Command::Flush => None,
             Command::Pause(_) => None,
             Command::Resume(_) => None,
             Command::DropStreamingJobs {
@@ -338,8 +332,15 @@ impl Command {
                 );
                 let mut changes: HashMap<_, _> = info
                     .new_fragment_info()
-                    .map(|(fragment_id, info)| {
-                        (fragment_id, CommandFragmentChanges::NewFragment(info))
+                    .map(|(fragment_id, fragment_info)| {
+                        (
+                            fragment_id,
+                            CommandFragmentChanges::NewFragment(
+                                info.streaming_job.database_id().into(),
+                                info.streaming_job.id().into(),
+                                fragment_info,
+                            ),
+                        )
                     })
                     .collect();
 
@@ -398,7 +399,7 @@ impl Command {
 
     pub fn need_checkpoint(&self) -> bool {
         // todo! Reviewing the flow of different command to reduce the amount of checkpoint
-        !matches!(self, Command::Plain(None) | Command::Resume(_))
+        !matches!(self, Command::Resume(_))
     }
 }
 
@@ -439,22 +440,15 @@ pub struct CommandContext {
     pub node_map: HashMap<WorkerId, PbWorkerNode>,
     pub subscription_info: InflightSubscriptionInfo,
 
-    pub prev_epoch: TracedEpoch,
-    pub curr_epoch: TracedEpoch,
+    pub barrier_info: BarrierInfo,
 
     pub table_ids_to_commit: HashSet<TableId>,
 
-    pub current_paused_reason: Option<PausedReason>,
-
-    pub command: Command,
-
-    pub kind: BarrierKind,
-
-    barrier_manager_context: GlobalBarrierManagerContext,
+    pub command: Option<Command>,
 
     /// The tracing span of this command.
     ///
-    /// Differs from [`TracedEpoch`], this span focuses on the lifetime of the corresponding
+    /// Differs from [`crate::barrier::TracedEpoch`], this span focuses on the lifetime of the corresponding
     /// barrier, including the process of waiting for the barrier to be sent, flowing through the
     /// stream graph on compute nodes, and finishing its `post_collect` stuffs.
     pub _span: tracing::Span,
@@ -463,9 +457,7 @@ pub struct CommandContext {
 impl std::fmt::Debug for CommandContext {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("CommandContext")
-            .field("prev_epoch", &self.prev_epoch.value().0)
-            .field("curr_epoch", &self.curr_epoch.value().0)
-            .field("kind", &self.kind)
+            .field("barrier_info", &self.barrier_info)
             .field("command", &self.command)
             .finish()
     }
@@ -474,26 +466,18 @@ impl std::fmt::Debug for CommandContext {
 impl CommandContext {
     pub(super) fn new(
         node_map: HashMap<WorkerId, PbWorkerNode>,
+        barrier_info: BarrierInfo,
         subscription_info: InflightSubscriptionInfo,
-        prev_epoch: TracedEpoch,
-        curr_epoch: TracedEpoch,
         table_ids_to_commit: HashSet<TableId>,
-        current_paused_reason: Option<PausedReason>,
-        command: Command,
-        kind: BarrierKind,
-        barrier_manager_context: GlobalBarrierManagerContext,
+        command: Option<Command>,
         span: tracing::Span,
     ) -> Self {
         Self {
             node_map,
             subscription_info,
-            prev_epoch,
-            curr_epoch,
+            barrier_info,
             table_ids_to_commit,
-            current_paused_reason,
             command,
-            kind,
-            barrier_manager_context,
             _span: span,
         }
     }
@@ -501,10 +485,10 @@ impl CommandContext {
 
 impl Command {
     /// Generate a mutation for the given command.
-    pub fn to_mutation(&self, current_paused_reason: Option<&PausedReason>) -> Option<Mutation> {
+    pub fn to_mutation(&self, current_paused_reason: Option<PausedReason>) -> Option<Mutation> {
         let mutation =
             match self {
-                Command::Plain(mutation) => mutation.clone(),
+                Command::Flush => None,
 
                 Command::Pause(_) => {
                     // Only pause when the cluster is not already paused.
@@ -517,7 +501,7 @@ impl Command {
 
                 Command::Resume(reason) => {
                     // Only resume when the cluster is paused with the same reason.
-                    if current_paused_reason == Some(reason) {
+                    if current_paused_reason == Some(*reason) {
                         Some(Mutation::Resume(ResumeMutation {}))
                     } else {
                         None
@@ -903,41 +887,35 @@ impl Command {
             ..Default::default()
         }))
     }
-}
-
-impl CommandContext {
-    pub fn to_mutation(&self) -> Option<Mutation> {
-        self.command
-            .to_mutation(self.current_paused_reason.as_ref())
-    }
 
     /// Returns the paused reason after executing the current command.
-    pub fn next_paused_reason(&self) -> Option<PausedReason> {
-        match &self.command {
-            Command::Pause(reason) => {
+    pub fn next_paused_reason(
+        this: Option<&Self>,
+        current_paused_reason: Option<PausedReason>,
+    ) -> Option<PausedReason> {
+        match this {
+            Some(Command::Pause(reason)) => {
                 // Only pause when the cluster is not already paused.
-                if self.current_paused_reason.is_none() {
+                if current_paused_reason.is_none() {
                     Some(*reason)
                 } else {
-                    self.current_paused_reason
+                    current_paused_reason
                 }
             }
 
-            Command::Resume(reason) => {
+            Some(Command::Resume(reason)) => {
                 // Only resume when the cluster is paused with the same reason.
-                if self.current_paused_reason == Some(*reason) {
+                if current_paused_reason == Some(*reason) {
                     None
                 } else {
-                    self.current_paused_reason
+                    current_paused_reason
                 }
             }
 
-            _ => self.current_paused_reason,
+            _ => current_paused_reason,
         }
     }
-}
 
-impl Command {
     /// For `CancelStreamingJob`, returns the table id of the target table.
     pub fn table_to_cancel(&self) -> Option<TableId> {
         match self {
@@ -948,7 +926,10 @@ impl Command {
 }
 
 impl CommandContext {
-    pub async fn wait_epoch_commit(&self) -> MetaResult<()> {
+    pub async fn wait_epoch_commit(
+        &self,
+        barrier_manager_context: &GlobalBarrierWorkerContextImpl,
+    ) -> MetaResult<()> {
         let table_id = self.table_ids_to_commit.iter().next().cloned();
         // try wait epoch on an existing random table id
         let Some(table_id) = table_id else {
@@ -956,14 +937,13 @@ impl CommandContext {
             return Ok(());
         };
         let futures = self.node_map.values().map(|worker_node| async {
-            let client = self
-                .barrier_manager_context
+            let client = barrier_manager_context
                 .env
                 .stream_client_pool()
                 .get(worker_node)
                 .await?;
             let request = WaitEpochCommitRequest {
-                epoch: self.prev_epoch.value().0,
+                epoch: self.barrier_info.prev_epoch(),
                 table_id: table_id.table_id,
             };
             client.wait_epoch_commit(request).await
@@ -976,9 +956,15 @@ impl CommandContext {
 
     /// Do some stuffs after barriers are collected and the new storage version is committed, for
     /// the given command.
-    pub async fn post_collect(&self) -> MetaResult<()> {
-        match &self.command {
-            Command::Plain(_) => {}
+    pub async fn post_collect(
+        &self,
+        barrier_manager_context: &GlobalBarrierWorkerContextImpl,
+    ) -> MetaResult<()> {
+        let Some(command) = &self.command else {
+            return Ok(());
+        };
+        match command {
+            Command::Flush => {}
 
             Command::Throttle(_) => {}
 
@@ -988,18 +974,18 @@ impl CommandContext {
                     // storage version with this epoch is synced to all compute nodes before the
                     // execution of the next command of `Update`, as some newly created operators
                     // may immediately initialize their states on that barrier.
-                    self.wait_epoch_commit().await?;
+                    self.wait_epoch_commit(barrier_manager_context).await?;
                 }
             }
 
             Command::Resume(_) => {}
 
             Command::SourceSplitAssignment(split_assignment) => {
-                self.barrier_manager_context
+                barrier_manager_context
                     .metadata_manager
                     .update_actor_splits_by_split_assignment(split_assignment)
                     .await?;
-                self.barrier_manager_context
+                barrier_manager_context
                     .source_manager
                     .apply_source_change(None, None, Some(split_assignment.clone()), None)
                     .await;
@@ -1009,7 +995,7 @@ impl CommandContext {
                 unregistered_state_table_ids,
                 ..
             } => {
-                self.barrier_manager_context
+                barrier_manager_context
                     .hummock_manager
                     .unregister_table_ids(unregistered_state_table_ids.iter().cloned())
                     .await?;
@@ -1027,7 +1013,7 @@ impl CommandContext {
                 // It won't clean the tables on failure,
                 // since the failure could be recoverable.
                 // As such it needs to be handled here.
-                self.barrier_manager_context
+                barrier_manager_context
                     .hummock_manager
                     .unregister_table_ids(table_fragments.all_table_ids().map(TableId::new))
                     .await?;
@@ -1040,7 +1026,7 @@ impl CommandContext {
                     init_split_assignment,
                     ..
                 } = info;
-                self.barrier_manager_context
+                barrier_manager_context
                     .metadata_manager
                     .catalog_controller
                     .post_collect_table_fragments(
@@ -1058,7 +1044,7 @@ impl CommandContext {
                     ..
                 }) = job_type
                 {
-                    self.barrier_manager_context
+                    barrier_manager_context
                         .metadata_manager
                         .catalog_controller
                         .post_collect_table_fragments(
@@ -1073,7 +1059,7 @@ impl CommandContext {
                 // Extract the fragments that include source operators.
                 let source_fragments = table_fragments.stream_source_fragments();
                 let backfill_fragments = table_fragments.source_backfill_fragments()?;
-                self.barrier_manager_context
+                barrier_manager_context
                     .source_manager
                     .apply_source_change(
                         Some(source_fragments),
@@ -1088,7 +1074,7 @@ impl CommandContext {
                 table_parallelism,
                 ..
             } => {
-                self.barrier_manager_context
+                barrier_manager_context
                     .scale_controller
                     .post_apply_reschedule(reschedules, table_parallelism)
                     .await?;
@@ -1102,7 +1088,7 @@ impl CommandContext {
                 ..
             }) => {
                 // Update actors and actor_dispatchers for new table fragments.
-                self.barrier_manager_context
+                barrier_manager_context
                     .metadata_manager
                     .catalog_controller
                     .post_collect_table_fragments(
@@ -1114,14 +1100,14 @@ impl CommandContext {
                     .await?;
 
                 // Apply the split changes in source manager.
-                self.barrier_manager_context
+                barrier_manager_context
                     .source_manager
                     .drop_source_fragments_vec(std::slice::from_ref(old_table_fragments))
                     .await;
                 let source_fragments = new_table_fragments.stream_source_fragments();
                 // XXX: is it possible to have backfill fragments here?
                 let backfill_fragments = new_table_fragments.source_backfill_fragments()?;
-                self.barrier_manager_context
+                barrier_manager_context
                     .source_manager
                     .apply_source_change(
                         Some(source_fragments),
@@ -1135,7 +1121,7 @@ impl CommandContext {
             Command::CreateSubscription {
                 subscription_id, ..
             } => {
-                self.barrier_manager_context
+                barrier_manager_context
                     .metadata_manager
                     .catalog_controller
                     .finish_create_subscription_catalog(*subscription_id)
@@ -1150,10 +1136,15 @@ impl CommandContext {
 
     pub fn get_truncate_epoch(&self, retention_second: u64) -> Epoch {
         let Some(truncate_timestamptz) = Timestamptz::from_secs(
-            self.prev_epoch.value().as_timestamptz().timestamp() - retention_second as i64,
+            self.barrier_info
+                .prev_epoch
+                .value()
+                .as_timestamptz()
+                .timestamp()
+                - retention_second as i64,
         ) else {
-            warn!(retention_second, prev_epoch = ?self.prev_epoch.value(), "invalid retention second value");
-            return self.prev_epoch.value();
+            warn!(retention_second, prev_epoch = ?self.barrier_info.prev_epoch.value(), "invalid retention second value");
+            return self.barrier_info.prev_epoch.value();
         };
         Epoch::from_unix_millis(truncate_timestamptz.timestamp_millis() as u64)
     }
diff --git a/src/meta/src/barrier/creating_job/barrier_control.rs b/src/meta/src/barrier/creating_job/barrier_control.rs
index e1d965c969372..7d238c37d4ebb 100644
--- a/src/meta/src/barrier/creating_job/barrier_control.rs
+++ b/src/meta/src/barrier/creating_job/barrier_control.rs
@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::Bound::{Excluded, Unbounded};
 use std::collections::{BTreeMap, HashSet, VecDeque};
 use std::mem::take;
+use std::ops::Bound::Unbounded;
+use std::ops::{Bound, RangeBounds};
 use std::time::Instant;
 
 use prometheus::HistogramTimer;
@@ -24,24 +25,15 @@ use risingwave_meta_model::WorkerId;
 use risingwave_pb::stream_service::BarrierCompleteResponse;
 use tracing::debug;
 
-use crate::rpc::metrics::MetaMetrics;
-
-#[derive(Debug)]
-pub(super) enum CreatingStreamingJobBarrierType {
-    Snapshot,
-    LogStore,
-    Upstream,
-}
+use crate::rpc::metrics::GLOBAL_META_METRICS;
 
 #[derive(Debug)]
 struct CreatingStreamingJobEpochState {
     epoch: u64,
     node_to_collect: HashSet<WorkerId>,
     resps: Vec<BarrierCompleteResponse>,
-    upstream_epoch_to_notify: Option<u64>,
     is_checkpoint: bool,
     enqueue_time: Instant,
-    barrier_type: CreatingStreamingJobBarrierType,
 }
 
 #[derive(Debug)]
@@ -49,47 +41,43 @@ pub(super) struct CreatingStreamingJobBarrierControl {
     table_id: TableId,
     // key is prev_epoch of barrier
     inflight_barrier_queue: BTreeMap<u64, CreatingStreamingJobEpochState>,
+    backfill_epoch: u64,
     initial_epoch: Option<u64>,
     max_collected_epoch: Option<u64>,
-    max_attached_epoch: Option<u64>,
-    // newer epoch at the front. should all be checkpoint barrier
+    // newer epoch at the front.
     pending_barriers_to_complete: VecDeque<CreatingStreamingJobEpochState>,
     completing_barrier: Option<(CreatingStreamingJobEpochState, HistogramTimer)>,
 
     // metrics
     consuming_snapshot_barrier_latency: LabelGuardedHistogram<2>,
     consuming_log_store_barrier_latency: LabelGuardedHistogram<2>,
-    consuming_upstream_barrier_latency: LabelGuardedHistogram<2>,
 
     wait_commit_latency: LabelGuardedHistogram<1>,
     inflight_barrier_num: LabelGuardedIntGauge<1>,
 }
 
 impl CreatingStreamingJobBarrierControl {
-    pub(super) fn new(table_id: TableId, metrics: &MetaMetrics) -> Self {
+    pub(super) fn new(table_id: TableId, backfill_epoch: u64) -> Self {
         let table_id_str = format!("{}", table_id.table_id);
         Self {
             table_id,
             inflight_barrier_queue: Default::default(),
+            backfill_epoch,
             initial_epoch: None,
             max_collected_epoch: None,
-            max_attached_epoch: None,
             pending_barriers_to_complete: Default::default(),
             completing_barrier: None,
 
-            consuming_snapshot_barrier_latency: metrics
+            consuming_snapshot_barrier_latency: GLOBAL_META_METRICS
                 .snapshot_backfill_barrier_latency
                 .with_guarded_label_values(&[&table_id_str, "consuming_snapshot"]),
-            consuming_log_store_barrier_latency: metrics
+            consuming_log_store_barrier_latency: GLOBAL_META_METRICS
                 .snapshot_backfill_barrier_latency
                 .with_guarded_label_values(&[&table_id_str, "consuming_log_store"]),
-            consuming_upstream_barrier_latency: metrics
-                .snapshot_backfill_barrier_latency
-                .with_guarded_label_values(&[&table_id_str, "consuming_upstream"]),
-            wait_commit_latency: metrics
+            wait_commit_latency: GLOBAL_META_METRICS
                 .snapshot_backfill_wait_commit_latency
                 .with_guarded_label_values(&[&table_id_str]),
-            inflight_barrier_num: metrics
+            inflight_barrier_num: GLOBAL_META_METRICS
                 .snapshot_backfill_inflight_barrier_num
                 .with_guarded_label_values(&[&table_id_str]),
         }
@@ -127,7 +115,6 @@ impl CreatingStreamingJobBarrierControl {
         epoch: u64,
         node_to_collect: HashSet<WorkerId>,
         is_checkpoint: bool,
-        barrier_type: CreatingStreamingJobBarrierType,
     ) {
         debug!(
             epoch,
@@ -142,17 +129,12 @@ impl CreatingStreamingJobBarrierControl {
         if let Some(latest_epoch) = self.latest_epoch() {
             assert!(epoch > latest_epoch, "{} {}", epoch, latest_epoch);
         }
-        if let Some(max_attached_epoch) = self.max_attached_epoch {
-            assert!(epoch > max_attached_epoch);
-        }
         let epoch_state = CreatingStreamingJobEpochState {
             epoch,
             node_to_collect,
             resps: vec![],
-            upstream_epoch_to_notify: None,
             is_checkpoint,
             enqueue_time: Instant::now(),
-            barrier_type,
         };
         if epoch_state.node_to_collect.is_empty() && self.inflight_barrier_queue.is_empty() {
             self.add_collected(epoch_state);
@@ -163,41 +145,6 @@ impl CreatingStreamingJobBarrierControl {
             .set(self.inflight_barrier_queue.len() as _);
     }
 
-    pub(super) fn unattached_epochs(&self) -> impl Iterator<Item = (u64, bool)> + '_ {
-        let range_start = if let Some(max_attached_epoch) = self.max_attached_epoch {
-            Excluded(max_attached_epoch)
-        } else {
-            Unbounded
-        };
-        self.inflight_barrier_queue
-            .range((range_start, Unbounded))
-            .map(|(epoch, state)| (*epoch, state.is_checkpoint))
-    }
-
-    /// Attach an `upstream_epoch` to the `epoch` of the creating job.
-    ///
-    /// The `upstream_epoch` won't be completed until the `epoch` of the creating job is completed so that
-    /// the `upstream_epoch` should wait for the progress of creating job, and we can ensure that the downstream
-    /// creating job can eventually catch up with the upstream.
-    pub(super) fn attach_upstream_epoch(&mut self, epoch: u64, upstream_epoch: u64) {
-        debug!(
-            epoch,
-            upstream_epoch,
-            table_id = ?self.table_id.table_id,
-            "attach epoch"
-        );
-        if let Some(max_attached_epoch) = self.max_attached_epoch {
-            assert!(epoch > max_attached_epoch);
-        }
-        self.max_attached_epoch = Some(epoch);
-        let epoch_state = self
-            .inflight_barrier_queue
-            .get_mut(&epoch)
-            .expect("should exist");
-        assert!(epoch_state.upstream_epoch_to_notify.is_none());
-        epoch_state.upstream_epoch_to_notify = Some(upstream_epoch);
-    }
-
     pub(super) fn collect(
         &mut self,
         epoch: u64,
@@ -228,46 +175,47 @@ impl CreatingStreamingJobBarrierControl {
             .set(self.inflight_barrier_queue.len() as _);
     }
 
-    #[expect(clippy::type_complexity)]
-    /// Return (`upstream_epochs_to_notify`, Some((epoch, resps, `is_first_commit`)))
+    /// Return Some((epoch, resps, `is_first_commit`))
     ///
-    /// `upstream_epochs_to_notify` is the upstream epochs of non-checkpoint barriers to be notified about barrier completing.
-    /// These non-checkpoint barriers does not need to call `commit_epoch` and therefore can be completed as long as collected.
+    /// Only epoch within the `epoch_end_bound` can be started.
+    /// Usually `epoch_end_bound` is the upstream committed epoch. This is to ensure that
+    /// the creating job won't have higher committed epoch than the upstream.
     pub(super) fn start_completing(
         &mut self,
-    ) -> (Vec<u64>, Option<(u64, Vec<BarrierCompleteResponse>, bool)>) {
-        if self.completing_barrier.is_some() {
-            return (vec![], None);
-        }
-        let mut upstream_epochs_to_notify = vec![];
-        while let Some(mut epoch_state) = self.pending_barriers_to_complete.pop_back() {
+        epoch_end_bound: Bound<u64>,
+    ) -> Option<(u64, Vec<BarrierCompleteResponse>, bool)> {
+        assert!(self.completing_barrier.is_none());
+        let epoch_range: (Bound<u64>, Bound<u64>) = (Unbounded, epoch_end_bound);
+        while let Some(epoch_state) = self.pending_barriers_to_complete.back()
+            && epoch_range.contains(&epoch_state.epoch)
+        {
+            let mut epoch_state = self
+                .pending_barriers_to_complete
+                .pop_back()
+                .expect("non-empty");
             let epoch = epoch_state.epoch;
             let is_first = self.initial_epoch.expect("should have set") == epoch;
             if is_first {
                 assert!(epoch_state.is_checkpoint);
             } else if !epoch_state.is_checkpoint {
-                if let Some(upstream_epoch) = epoch_state.upstream_epoch_to_notify {
-                    upstream_epochs_to_notify.push(upstream_epoch);
-                }
                 continue;
             }
 
             let resps = take(&mut epoch_state.resps);
             self.completing_barrier = Some((epoch_state, self.wait_commit_latency.start_timer()));
-            return (upstream_epochs_to_notify, Some((epoch, resps, is_first)));
+            return Some((epoch, resps, is_first));
         }
-        (upstream_epochs_to_notify, None)
+        None
     }
 
     /// Ack on completing a checkpoint barrier.
     ///
     /// Return the upstream epoch to be notified when there is any.
-    pub(super) fn ack_completed(&mut self, completed_epoch: u64) -> Option<u64> {
+    pub(super) fn ack_completed(&mut self, completed_epoch: u64) {
         let (epoch_state, wait_commit_timer) =
             self.completing_barrier.take().expect("should exist");
         wait_commit_timer.observe_duration();
         assert_eq!(epoch_state.epoch, completed_epoch);
-        epoch_state.upstream_epoch_to_notify
     }
 
     fn add_collected(&mut self, epoch_state: CreatingStreamingJobEpochState) {
@@ -280,10 +228,10 @@ impl CreatingStreamingJobBarrierControl {
         }
         self.max_collected_epoch = Some(epoch_state.epoch);
         let barrier_latency = epoch_state.enqueue_time.elapsed().as_secs_f64();
-        let barrier_latency_metrics = match &epoch_state.barrier_type {
-            CreatingStreamingJobBarrierType::Snapshot => &self.consuming_snapshot_barrier_latency,
-            CreatingStreamingJobBarrierType::LogStore => &self.consuming_log_store_barrier_latency,
-            CreatingStreamingJobBarrierType::Upstream => &self.consuming_upstream_barrier_latency,
+        let barrier_latency_metrics = if epoch_state.epoch < self.backfill_epoch {
+            &self.consuming_snapshot_barrier_latency
+        } else {
+            &self.consuming_log_store_barrier_latency
         };
         barrier_latency_metrics.observe(barrier_latency);
         self.pending_barriers_to_complete.push_front(epoch_state);
diff --git a/src/meta/src/barrier/creating_job/mod.rs b/src/meta/src/barrier/creating_job/mod.rs
index 42e416d737881..522b6a3563eeb 100644
--- a/src/meta/src/barrier/creating_job/mod.rs
+++ b/src/meta/src/barrier/creating_job/mod.rs
@@ -17,33 +17,27 @@ mod status;
 
 use std::cmp::max;
 use std::collections::HashMap;
-use std::mem::take;
-use std::sync::Arc;
-use std::time::Duration;
+use std::ops::Bound::{Excluded, Unbounded};
 
-use prometheus::HistogramTimer;
-use risingwave_common::metrics::{LabelGuardedHistogram, LabelGuardedIntGauge};
-use risingwave_common::util::epoch::Epoch;
+use risingwave_common::catalog::TableId;
+use risingwave_common::metrics::LabelGuardedIntGauge;
 use risingwave_meta_model::WorkerId;
-use risingwave_pb::common::WorkerNode;
 use risingwave_pb::ddl_service::DdlProgress;
 use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::stream_plan::barrier_mutation::Mutation;
 use risingwave_pb::stream_service::BarrierCompleteResponse;
-use tracing::{debug, info};
+use tracing::info;
 
-use crate::barrier::command::CommandContext;
-use crate::barrier::creating_job::barrier_control::{
-    CreatingStreamingJobBarrierControl, CreatingStreamingJobBarrierType,
-};
+use crate::barrier::creating_job::barrier_control::CreatingStreamingJobBarrierControl;
 use crate::barrier::creating_job::status::{
     CreatingJobInjectBarrierInfo, CreatingStreamingJobStatus,
 };
-use crate::barrier::info::InflightGraphInfo;
+use crate::barrier::info::{BarrierInfo, InflightStreamingJobInfo};
 use crate::barrier::progress::CreateMviewProgressTracker;
 use crate::barrier::rpc::ControlStreamManager;
 use crate::barrier::{Command, CreateStreamingJobCommandInfo, SnapshotBackfillInfo};
-use crate::rpc::metrics::MetaMetrics;
+use crate::controller::fragment::InflightFragmentInfo;
+use crate::rpc::metrics::GLOBAL_META_METRICS;
 use crate::MetaResult;
 
 #[derive(Debug)]
@@ -52,11 +46,12 @@ pub(super) struct CreatingStreamingJobControl {
     pub(super) snapshot_backfill_info: SnapshotBackfillInfo,
     backfill_epoch: u64,
 
+    graph_info: InflightStreamingJobInfo,
+
     barrier_control: CreatingStreamingJobBarrierControl,
     status: CreatingStreamingJobStatus,
 
     upstream_lag: LabelGuardedIntGauge<1>,
-    upstream_wait_progress_latency: LabelGuardedHistogram<1>,
 }
 
 impl CreatingStreamingJobControl {
@@ -65,7 +60,6 @@ impl CreatingStreamingJobControl {
         snapshot_backfill_info: SnapshotBackfillInfo,
         backfill_epoch: u64,
         version_stat: &HummockVersionStats,
-        metrics: &MetaMetrics,
         initial_mutation: Mutation,
     ) -> Self {
         info!(
@@ -73,56 +67,50 @@ impl CreatingStreamingJobControl {
             definition = info.definition,
             "new creating job"
         );
+        let snapshot_backfill_actors = info.table_fragments.snapshot_backfill_actor_ids();
         let mut create_mview_tracker = CreateMviewProgressTracker::default();
         create_mview_tracker.update_tracking_jobs(Some((&info, None)), [], version_stat);
-        let fragment_info: HashMap<_, _> = info.new_fragment_info().collect();
+        let fragment_infos: HashMap<_, _> = info.new_fragment_info().collect();
 
         let table_id = info.table_fragments.table_id();
         let table_id_str = format!("{}", table_id.table_id);
 
         let actors_to_create = info.table_fragments.actors_to_create();
+        let graph_info = InflightStreamingJobInfo {
+            job_id: table_id,
+            database_id: info.streaming_job.database_id().into(),
+            fragment_infos,
+        };
 
         Self {
             info,
             snapshot_backfill_info,
-            barrier_control: CreatingStreamingJobBarrierControl::new(table_id, metrics),
+            barrier_control: CreatingStreamingJobBarrierControl::new(table_id, backfill_epoch),
             backfill_epoch,
+            graph_info,
             status: CreatingStreamingJobStatus::ConsumingSnapshot {
                 prev_epoch_fake_physical_time: 0,
-                pending_commands: vec![],
+                pending_upstream_barriers: vec![],
                 version_stats: version_stat.clone(),
                 create_mview_tracker,
-                graph_info: InflightGraphInfo::new(fragment_info),
+                snapshot_backfill_actors,
                 backfill_epoch,
                 pending_non_checkpoint_barriers: vec![],
                 initial_barrier_info: Some((actors_to_create, initial_mutation)),
             },
-            upstream_lag: metrics
+            upstream_lag: GLOBAL_META_METRICS
                 .snapshot_backfill_lag
                 .with_guarded_label_values(&[&table_id_str]),
-            upstream_wait_progress_latency: metrics
-                .snapshot_backfill_upstream_wait_progress_latency
-                .with_guarded_label_values(&[&table_id_str]),
         }
     }
 
-    pub(super) fn start_wait_progress_timer(&self) -> HistogramTimer {
-        self.upstream_wait_progress_latency.start_timer()
-    }
-
     pub(super) fn is_wait_on_worker(&self, worker_id: WorkerId) -> bool {
         self.barrier_control.is_wait_on_worker(worker_id)
-            || self
-                .status
-                .active_graph_info()
-                .map(|info| info.contains_worker(worker_id))
-                .unwrap_or(false)
-    }
-
-    pub(super) fn on_new_worker_node_map(&self, node_map: &HashMap<WorkerId, WorkerNode>) {
-        if let Some(info) = self.status.active_graph_info() {
-            info.on_new_worker_node_map(node_map)
-        }
+            || (self.status.is_finishing()
+                && InflightFragmentInfo::contains_worker(
+                    self.graph_info.fragment_infos(),
+                    worker_id,
+                ))
     }
 
     pub(super) fn gen_ddl_progress(&self) -> DdlProgress {
@@ -142,32 +130,15 @@ impl CreatingStreamingJobControl {
                 }
             }
             CreatingStreamingJobStatus::ConsumingLogStore {
-                start_consume_log_store_epoch,
+                log_store_progress_tracker,
                 ..
             } => {
-                let max_collected_epoch = max(
-                    self.barrier_control.max_collected_epoch().unwrap_or(0),
-                    self.backfill_epoch,
-                );
-                let lag = Duration::from_millis(
-                    Epoch(*start_consume_log_store_epoch)
-                        .physical_time()
-                        .saturating_sub(Epoch(max_collected_epoch).physical_time()),
-                );
                 format!(
-                    "LogStore [remain lag: {:?}, epoch cnt: {}]",
-                    lag,
-                    self.barrier_control.inflight_barrier_count()
+                    "LogStore [{}]",
+                    log_store_progress_tracker.gen_ddl_progress()
                 )
             }
-            CreatingStreamingJobStatus::ConsumingUpstream { .. } => {
-                format!(
-                    "Upstream [unattached: {}, epoch cnt: {}]",
-                    self.barrier_control.unattached_epochs().count(),
-                    self.barrier_control.inflight_barrier_count(),
-                )
-            }
-            CreatingStreamingJobStatus::Finishing { .. } => {
+            CreatingStreamingJobStatus::Finishing(_) => {
                 format!(
                     "Finishing [epoch count: {}]",
                     self.barrier_control.inflight_barrier_count()
@@ -182,101 +153,70 @@ impl CreatingStreamingJobControl {
     }
 
     pub(super) fn pinned_upstream_log_epoch(&self) -> Option<u64> {
-        let stop_consume_log_store_epoch = match &self.status {
-            CreatingStreamingJobStatus::ConsumingSnapshot { .. }
-            | CreatingStreamingJobStatus::ConsumingLogStore { .. } => None,
-            CreatingStreamingJobStatus::ConsumingUpstream {
-                start_consume_upstream_epoch,
-                ..
-            }
-            | CreatingStreamingJobStatus::Finishing {
-                start_consume_upstream_epoch,
-                ..
-            } => Some(*start_consume_upstream_epoch),
-        };
-        if let Some(max_collected_epoch) = self.barrier_control.max_collected_epoch() {
-            if max_collected_epoch < self.backfill_epoch {
-                Some(self.backfill_epoch)
-            } else if let Some(stop_consume_log_store_epoch) = stop_consume_log_store_epoch
-                && max_collected_epoch >= stop_consume_log_store_epoch
-            {
-                None
-            } else {
-                Some(max_collected_epoch)
-            }
+        if self.status.is_finishing() {
+            None
         } else {
-            Some(self.backfill_epoch)
+            // TODO: when supporting recoverable snapshot backfill, we should use the max epoch that has committed
+            Some(max(
+                self.barrier_control.max_collected_epoch().unwrap_or(0),
+                self.backfill_epoch,
+            ))
         }
     }
 
-    pub(super) fn may_inject_fake_barrier(
-        &mut self,
+    fn inject_barrier(
+        table_id: TableId,
         control_stream_manager: &mut ControlStreamManager,
-        upstream_prev_epoch: u64,
-        is_checkpoint: bool,
+        barrier_control: &mut CreatingStreamingJobBarrierControl,
+        pre_applied_graph_info: &InflightStreamingJobInfo,
+        applied_graph_info: Option<&InflightStreamingJobInfo>,
+        CreatingJobInjectBarrierInfo {
+            barrier_info,
+            new_actors,
+            mutation,
+        }: CreatingJobInjectBarrierInfo,
     ) -> MetaResult<()> {
-        if let Some((barriers_to_inject, graph_info)) =
-            self.status.may_inject_fake_barrier(is_checkpoint)
-        {
-            if let Some(graph_info) = graph_info {
-                info!(
-                    table_id = self.info.table_fragments.table_id().table_id,
-                    upstream_prev_epoch, "start consuming log store"
-                );
-                self.status = CreatingStreamingJobStatus::ConsumingLogStore {
-                    graph_info,
-                    start_consume_log_store_epoch: upstream_prev_epoch,
-                };
-            }
-            let graph_info = self
-                .status
-                .active_graph_info()
-                .expect("must exist when having barriers to inject");
-            let table_id = self.info.table_fragments.table_id();
-            for CreatingJobInjectBarrierInfo {
-                curr_epoch,
-                prev_epoch,
-                kind,
-                new_actors,
-                mutation,
-            } in barriers_to_inject
-            {
-                let node_to_collect = control_stream_manager.inject_barrier(
-                    Some(table_id),
-                    mutation,
-                    (&curr_epoch, &prev_epoch),
-                    &kind,
-                    graph_info,
-                    Some(graph_info),
-                    new_actors,
-                    vec![],
-                    vec![],
-                )?;
-                self.barrier_control.enqueue_epoch(
-                    prev_epoch.value().0,
-                    node_to_collect,
-                    kind.is_checkpoint(),
-                    CreatingStreamingJobBarrierType::Snapshot,
-                );
-            }
-        }
+        let node_to_collect = control_stream_manager.inject_barrier(
+            Some(table_id),
+            mutation,
+            &barrier_info,
+            pre_applied_graph_info.fragment_infos(),
+            applied_graph_info
+                .map(|graph_info| graph_info.fragment_infos())
+                .into_iter()
+                .flatten(),
+            new_actors,
+            vec![],
+            vec![],
+        )?;
+        barrier_control.enqueue_epoch(
+            barrier_info.prev_epoch(),
+            node_to_collect,
+            barrier_info.kind.is_checkpoint(),
+        );
         Ok(())
     }
 
     pub(super) fn on_new_command(
         &mut self,
         control_stream_manager: &mut ControlStreamManager,
-        command_ctx: &Arc<CommandContext>,
-    ) -> MetaResult<Option<Option<InflightGraphInfo>>> {
+        command: Option<&Command>,
+        barrier_info: &BarrierInfo,
+    ) -> MetaResult<()> {
         let table_id = self.info.table_fragments.table_id();
-        let start_consume_upstream = if let Command::MergeSnapshotBackfillStreamingJobs(
-            jobs_to_merge,
-        ) = &command_ctx.command
-        {
-            jobs_to_merge.contains_key(&table_id)
-        } else {
-            false
-        };
+        let start_consume_upstream =
+            if let Some(Command::MergeSnapshotBackfillStreamingJobs(jobs_to_merge)) = command {
+                jobs_to_merge.contains_key(&table_id)
+            } else {
+                false
+            };
+        if start_consume_upstream {
+            info!(
+                table_id = self.info.table_fragments.table_id().table_id,
+                prev_epoch = barrier_info.prev_epoch(),
+                "start consuming upstream"
+            );
+        }
         let progress_epoch =
             if let Some(max_collected_epoch) = self.barrier_control.max_collected_epoch() {
                 max(max_collected_epoch, self.backfill_epoch)
@@ -284,131 +224,30 @@ impl CreatingStreamingJobControl {
                 self.backfill_epoch
             };
         self.upstream_lag.set(
-            command_ctx
+            barrier_info
                 .prev_epoch
                 .value()
                 .0
                 .saturating_sub(progress_epoch) as _,
         );
-        let graph_to_finish = match &mut self.status {
-            CreatingStreamingJobStatus::ConsumingSnapshot {
-                pending_commands, ..
-            } => {
-                assert!(
-                    !start_consume_upstream,
-                    "should not start consuming upstream for a job that are consuming snapshot"
-                );
-                pending_commands.push(command_ctx.clone());
-                None
-            }
-            CreatingStreamingJobStatus::ConsumingLogStore { graph_info, .. } => {
-                let node_to_collect = control_stream_manager.inject_barrier(
-                    Some(table_id),
-                    if start_consume_upstream {
-                        // erase the mutation on upstream except the last command
-                        command_ctx.to_mutation()
-                    } else {
-                        None
-                    },
-                    (&command_ctx.curr_epoch, &command_ctx.prev_epoch),
-                    &command_ctx.kind,
-                    graph_info,
-                    Some(graph_info),
-                    None,
-                    vec![],
-                    vec![],
-                )?;
-                self.barrier_control.enqueue_epoch(
-                    command_ctx.prev_epoch.value().0,
-                    node_to_collect,
-                    command_ctx.kind.is_checkpoint(),
-                    CreatingStreamingJobBarrierType::LogStore,
-                );
-                let prev_epoch = command_ctx.prev_epoch.value().0;
+        if let Some(barrier_to_inject) = self
+            .status
+            .on_new_upstream_epoch(barrier_info, start_consume_upstream)
+        {
+            Self::inject_barrier(
+                self.info.table_fragments.table_id(),
+                control_stream_manager,
+                &mut self.barrier_control,
+                &self.graph_info,
                 if start_consume_upstream {
-                    let graph_info = take(graph_info);
-                    info!(
-                        table_id = self.info.table_fragments.table_id().table_id,
-                        prev_epoch, "start consuming upstream"
-                    );
-                    self.status = CreatingStreamingJobStatus::ConsumingUpstream {
-                        start_consume_upstream_epoch: prev_epoch,
-                        graph_info,
-                    };
-                }
-                None
-            }
-            CreatingStreamingJobStatus::ConsumingUpstream {
-                start_consume_upstream_epoch,
-                graph_info,
-            } => {
-                assert!(
-                    !start_consume_upstream,
-                    "should not start consuming upstream for a job again"
-                );
-
-                let should_finish = command_ctx.kind.is_checkpoint()
-                    && self.barrier_control.unattached_epochs().next().is_none();
-                let node_to_collect = control_stream_manager.inject_barrier(
-                    Some(table_id),
-                    // do not send the upstream barrier mutation because in `ConsumingUpstream` stage,
-                    // barriers are still injected and collected independently on the creating jobs.
-                    None,
-                    (&command_ctx.curr_epoch, &command_ctx.prev_epoch),
-                    &command_ctx.kind,
-                    graph_info,
-                    if should_finish {
-                        None
-                    } else {
-                        Some(graph_info)
-                    },
-                    None,
-                    vec![],
-                    vec![],
-                )?;
-                let prev_epoch = command_ctx.prev_epoch.value().0;
-                self.barrier_control.enqueue_epoch(
-                    prev_epoch,
-                    node_to_collect,
-                    command_ctx.kind.is_checkpoint(),
-                    CreatingStreamingJobBarrierType::Upstream,
-                );
-                let graph_info = if should_finish {
-                    info!(prev_epoch, table_id = ?self.info.table_fragments.table_id(), "mark as finishing");
-                    self.barrier_control
-                        .attach_upstream_epoch(prev_epoch, prev_epoch);
-                    let graph_info = take(graph_info);
-                    self.status = CreatingStreamingJobStatus::Finishing {
-                        start_consume_upstream_epoch: *start_consume_upstream_epoch,
-                    };
-                    Some(Some(graph_info))
+                    None
                 } else {
-                    let mut unattached_epochs_iter = self.barrier_control.unattached_epochs();
-                    let mut epoch_to_attach = unattached_epochs_iter.next().expect("non-empty").0;
-                    let mut remain_count = 5;
-                    while remain_count > 0
-                        && let Some((epoch, _)) = unattached_epochs_iter.next()
-                    {
-                        remain_count -= 1;
-                        epoch_to_attach = epoch;
-                    }
-                    drop(unattached_epochs_iter);
-                    self.barrier_control
-                        .attach_upstream_epoch(epoch_to_attach, prev_epoch);
-                    Some(None)
-                };
-
-                graph_info
-            }
-            CreatingStreamingJobStatus::Finishing { .. } => {
-                assert!(
-                    !start_consume_upstream,
-                    "should not start consuming upstream for a job again"
-                );
-                None
-            }
-        };
-        Ok(graph_to_finish)
+                    Some(&self.graph_info)
+                },
+                barrier_to_inject,
+            )?;
+        }
+        Ok(())
     }
 
     pub(super) fn collect(
@@ -416,57 +255,99 @@ impl CreatingStreamingJobControl {
         epoch: u64,
         worker_id: WorkerId,
         resp: BarrierCompleteResponse,
-    ) {
-        self.status.update_progress(&resp.create_mview_progress);
+        control_stream_manager: &mut ControlStreamManager,
+    ) -> MetaResult<()> {
+        let prev_barriers_to_inject = self.status.update_progress(&resp.create_mview_progress);
         self.barrier_control.collect(epoch, worker_id, resp);
+        if let Some(prev_barriers_to_inject) = prev_barriers_to_inject {
+            let table_id = self.info.table_fragments.table_id();
+            for info in prev_barriers_to_inject {
+                Self::inject_barrier(
+                    table_id,
+                    control_stream_manager,
+                    &mut self.barrier_control,
+                    &self.graph_info,
+                    Some(&self.graph_info),
+                    info,
+                )?;
+            }
+        }
+        Ok(())
     }
 
-    pub(super) fn should_merge_to_upstream(&self) -> Option<InflightGraphInfo> {
-        if let (
-            CreatingStreamingJobStatus::ConsumingLogStore {
-                graph_info,
-                start_consume_log_store_epoch,
-            },
-            Some(max_collected_epoch),
-        ) = (&self.status, self.barrier_control.max_collected_epoch())
+    pub(super) fn should_merge_to_upstream(&self) -> Option<InflightStreamingJobInfo> {
+        if let CreatingStreamingJobStatus::ConsumingLogStore {
+            ref log_store_progress_tracker,
+        } = &self.status
+            && log_store_progress_tracker.is_finished()
         {
-            if max_collected_epoch >= *start_consume_log_store_epoch {
-                Some(graph_info.clone())
-            } else {
-                let lag = Duration::from_millis(
-                    Epoch(*start_consume_log_store_epoch).physical_time()
-                        - Epoch(max_collected_epoch).physical_time(),
-                );
-                debug!(
-                    ?lag,
-                    max_collected_epoch, start_consume_log_store_epoch, "wait consuming log store"
-                );
-                None
-            }
+            Some(self.graph_info.clone())
         } else {
             None
         }
     }
+}
+
+pub(super) enum CompleteJobType {
+    /// The first barrier
+    First,
+    Normal,
+    /// The last barrier to complete
+    Finished,
+}
 
-    #[expect(clippy::type_complexity)]
+impl CreatingStreamingJobControl {
     pub(super) fn start_completing(
         &mut self,
-    ) -> (Vec<u64>, Option<(u64, Vec<BarrierCompleteResponse>, bool)>) {
-        self.barrier_control.start_completing()
+        min_upstream_inflight_epoch: Option<u64>,
+    ) -> Option<(u64, Vec<BarrierCompleteResponse>, CompleteJobType)> {
+        let (finished_at_epoch, epoch_end_bound) = match &self.status {
+            CreatingStreamingJobStatus::Finishing(finish_at_epoch) => {
+                let epoch_end_bound = min_upstream_inflight_epoch
+                    .map(|upstream_epoch| {
+                        if upstream_epoch < *finish_at_epoch {
+                            Excluded(upstream_epoch)
+                        } else {
+                            Unbounded
+                        }
+                    })
+                    .unwrap_or(Unbounded);
+                (Some(*finish_at_epoch), epoch_end_bound)
+            }
+            CreatingStreamingJobStatus::ConsumingSnapshot { .. }
+            | CreatingStreamingJobStatus::ConsumingLogStore { .. } => (
+                None,
+                min_upstream_inflight_epoch
+                    .map(Excluded)
+                    .unwrap_or(Unbounded),
+            ),
+        };
+        self.barrier_control.start_completing(epoch_end_bound).map(
+            |(epoch, resps, is_first_commit)| {
+                let status = if let Some(finish_at_epoch) = finished_at_epoch {
+                    assert!(!is_first_commit);
+                    if epoch == finish_at_epoch {
+                        self.barrier_control.ack_completed(epoch);
+                        assert!(self.barrier_control.is_empty());
+                        CompleteJobType::Finished
+                    } else {
+                        CompleteJobType::Normal
+                    }
+                } else if is_first_commit {
+                    CompleteJobType::First
+                } else {
+                    CompleteJobType::Normal
+                };
+                (epoch, resps, status)
+            },
+        )
     }
 
-    pub(super) fn ack_completed(&mut self, completed_epoch: u64) -> Option<(u64, bool)> {
-        let upstream_epoch_to_notify = self.barrier_control.ack_completed(completed_epoch);
-        if let Some(upstream_epoch_to_notify) = upstream_epoch_to_notify {
-            Some((upstream_epoch_to_notify, self.is_finished()))
-        } else {
-            assert!(!self.is_finished());
-            None
-        }
+    pub(super) fn ack_completed(&mut self, completed_epoch: u64) {
+        self.barrier_control.ack_completed(completed_epoch);
     }
 
     pub(super) fn is_finished(&self) -> bool {
-        self.barrier_control.is_empty()
-            && matches!(&self.status, CreatingStreamingJobStatus::Finishing { .. })
+        self.barrier_control.is_empty() && self.status.is_finishing()
     }
 }
diff --git a/src/meta/src/barrier/creating_job/status.rs b/src/meta/src/barrier/creating_job/status.rs
index bfc5b4bbe7faf..f5d18400a98e1 100644
--- a/src/meta/src/barrier/creating_job/status.rs
+++ b/src/meta/src/barrier/creating_job/status.rs
@@ -12,30 +12,99 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
+use std::collections::hash_map::Entry;
+use std::collections::{HashMap, HashSet};
 use std::mem::take;
-use std::sync::Arc;
 
+use risingwave_common::hash::ActorId;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_meta_model::WorkerId;
 use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::stream_plan::barrier_mutation::Mutation;
 use risingwave_pb::stream_plan::StreamActor;
-use risingwave_pb::stream_service::barrier_complete_response::CreateMviewProgress;
+use risingwave_pb::stream_service::barrier_complete_response::{
+    CreateMviewProgress, PbCreateMviewProgress,
+};
+use tracing::warn;
 
-use crate::barrier::command::CommandContext;
-use crate::barrier::info::InflightGraphInfo;
 use crate::barrier::progress::CreateMviewProgressTracker;
-use crate::barrier::{BarrierKind, TracedEpoch};
+use crate::barrier::{BarrierInfo, BarrierKind, TracedEpoch};
+
+#[derive(Debug)]
+pub(super) struct CreateMviewLogStoreProgressTracker {
+    /// `actor_id` -> `pending_barrier_count`
+    ongoing_actors: HashMap<ActorId, usize>,
+    finished_actors: HashSet<ActorId>,
+}
+
+impl CreateMviewLogStoreProgressTracker {
+    fn new(actors: impl Iterator<Item = ActorId>, initial_pending_count: usize) -> Self {
+        Self {
+            ongoing_actors: HashMap::from_iter(actors.map(|actor| (actor, initial_pending_count))),
+            finished_actors: HashSet::new(),
+        }
+    }
+
+    pub(super) fn gen_ddl_progress(&self) -> String {
+        let sum = self.ongoing_actors.values().sum::<usize>() as f64;
+        let count = if self.ongoing_actors.is_empty() {
+            1
+        } else {
+            self.ongoing_actors.len()
+        } as f64;
+        let avg = sum / count;
+        format!(
+            "finished: {}/{}, avg epoch count {}",
+            self.finished_actors.len(),
+            self.ongoing_actors.len() + self.finished_actors.len(),
+            avg
+        )
+    }
+
+    fn update(&mut self, progress: impl IntoIterator<Item = &PbCreateMviewProgress>) {
+        for progress in progress {
+            match self.ongoing_actors.entry(progress.backfill_actor_id) {
+                Entry::Occupied(mut entry) => {
+                    if progress.done {
+                        entry.remove_entry();
+                        assert!(
+                            self.finished_actors.insert(progress.backfill_actor_id),
+                            "non-duplicate"
+                        );
+                    } else {
+                        *entry.get_mut() = progress.pending_barrier_num as _;
+                    }
+                }
+                Entry::Vacant(_) => {
+                    if cfg!(debug_assertions) {
+                        panic!(
+                            "reporting progress on non-inflight actor: {:?} {:?}",
+                            progress, self
+                        );
+                    } else {
+                        warn!(?progress, progress_tracker = ?self, "reporting progress on non-inflight actor");
+                    }
+                }
+            }
+        }
+    }
+
+    pub(super) fn is_finished(&self) -> bool {
+        self.ongoing_actors.is_empty()
+    }
+}
 
 #[derive(Debug)]
 pub(super) enum CreatingStreamingJobStatus {
+    /// The creating job is consuming upstream snapshot.
+    /// Will transit to `ConsumingLogStore` on `update_progress` when
+    /// the snapshot has been fully consumed after `update_progress`.
     ConsumingSnapshot {
         prev_epoch_fake_physical_time: u64,
-        pending_commands: Vec<Arc<CommandContext>>,
+        pending_upstream_barriers: Vec<BarrierInfo>,
         version_stats: HummockVersionStats,
         create_mview_tracker: CreateMviewProgressTracker,
-        graph_info: InflightGraphInfo,
+        snapshot_backfill_actors: HashSet<ActorId>,
         backfill_epoch: u64,
         /// The `prev_epoch` of pending non checkpoint barriers
         pending_non_checkpoint_barriers: Vec<u64>,
@@ -43,101 +112,151 @@ pub(super) enum CreatingStreamingJobStatus {
         /// Take the mutation out when injecting the first barrier
         initial_barrier_info: Option<(HashMap<WorkerId, Vec<StreamActor>>, Mutation)>,
     },
+    /// The creating job is consuming log store.
+    ///
+    /// Will transit to `Finishing` on `on_new_upstream_epoch` when `start_consume_upstream` is `true`.
     ConsumingLogStore {
-        graph_info: InflightGraphInfo,
-        start_consume_log_store_epoch: u64,
-    },
-    ConsumingUpstream {
-        start_consume_upstream_epoch: u64,
-        graph_info: InflightGraphInfo,
-    },
-    Finishing {
-        start_consume_upstream_epoch: u64,
+        log_store_progress_tracker: CreateMviewLogStoreProgressTracker,
     },
+    /// All backfill actors have started consuming upstream, and the job
+    /// will be finished when all previously injected barriers have been collected
+    /// Store the `prev_epoch` that will finish at.
+    Finishing(u64),
 }
 
 pub(super) struct CreatingJobInjectBarrierInfo {
-    pub curr_epoch: TracedEpoch,
-    pub prev_epoch: TracedEpoch,
-    pub kind: BarrierKind,
+    pub barrier_info: BarrierInfo,
     pub new_actors: Option<HashMap<WorkerId, Vec<StreamActor>>>,
     pub mutation: Option<Mutation>,
 }
 
 impl CreatingStreamingJobStatus {
-    pub(super) fn active_graph_info(&self) -> Option<&InflightGraphInfo> {
-        match self {
-            CreatingStreamingJobStatus::ConsumingSnapshot { graph_info, .. }
-            | CreatingStreamingJobStatus::ConsumingLogStore { graph_info, .. }
-            | CreatingStreamingJobStatus::ConsumingUpstream { graph_info, .. } => Some(graph_info),
-            CreatingStreamingJobStatus::Finishing { .. } => {
-                // when entering `Finishing`, the graph will have been added to the upstream graph,
-                // and therefore the separate graph info is inactive.
-                None
-            }
-        }
-    }
-
     pub(super) fn update_progress(
         &mut self,
         create_mview_progress: impl IntoIterator<Item = &CreateMviewProgress>,
-    ) {
-        if let Self::ConsumingSnapshot {
-            create_mview_tracker,
-            ref version_stats,
-            ..
-        } = self
-        {
-            create_mview_tracker.update_tracking_jobs(None, create_mview_progress, version_stats);
-        }
-    }
+    ) -> Option<Vec<CreatingJobInjectBarrierInfo>> {
+        match self {
+            Self::ConsumingSnapshot {
+                create_mview_tracker,
+                ref version_stats,
+                prev_epoch_fake_physical_time,
+                pending_upstream_barriers,
+                pending_non_checkpoint_barriers,
+                ref backfill_epoch,
+                initial_barrier_info,
+                ref snapshot_backfill_actors,
+                ..
+            } => {
+                create_mview_tracker.update_tracking_jobs(
+                    None,
+                    create_mview_progress,
+                    version_stats,
+                );
+                if create_mview_tracker.has_pending_finished_jobs() {
+                    let (new_actors, mutation) = match initial_barrier_info.take() {
+                        Some((new_actors, mutation)) => (Some(new_actors), Some(mutation)),
+                        None => (None, None),
+                    };
+                    assert!(initial_barrier_info.is_none());
+                    pending_non_checkpoint_barriers.push(*backfill_epoch);
 
-    /// return
-    /// - Some(vec[(`curr_epoch`, `prev_epoch`, `barrier_kind`)]) of barriers to newly inject
-    /// - Some(`graph_info`) when the status should transit to `ConsumingLogStore`
-    pub(super) fn may_inject_fake_barrier(
-        &mut self,
-        is_checkpoint: bool,
-    ) -> Option<(Vec<CreatingJobInjectBarrierInfo>, Option<InflightGraphInfo>)> {
-        if let CreatingStreamingJobStatus::ConsumingSnapshot {
-            prev_epoch_fake_physical_time,
-            pending_commands,
-            create_mview_tracker,
-            graph_info,
-            pending_non_checkpoint_barriers,
-            ref backfill_epoch,
-            initial_barrier_info,
-            ..
-        } = self
-        {
-            if create_mview_tracker.has_pending_finished_jobs() {
-                assert!(initial_barrier_info.is_none());
-                pending_non_checkpoint_barriers.push(*backfill_epoch);
-
-                let prev_epoch = Epoch::from_physical_time(*prev_epoch_fake_physical_time);
-                let barriers_to_inject =
-                    [CreatingJobInjectBarrierInfo {
-                        curr_epoch: TracedEpoch::new(Epoch(*backfill_epoch)),
-                        prev_epoch: TracedEpoch::new(prev_epoch),
-                        kind: BarrierKind::Checkpoint(take(pending_non_checkpoint_barriers)),
-                        new_actors: None,
-                        mutation: None,
+                    let prev_epoch = Epoch::from_physical_time(*prev_epoch_fake_physical_time);
+                    let barriers_to_inject: Vec<_> = [CreatingJobInjectBarrierInfo {
+                        barrier_info: BarrierInfo {
+                            curr_epoch: TracedEpoch::new(Epoch(*backfill_epoch)),
+                            prev_epoch: TracedEpoch::new(prev_epoch),
+                            kind: BarrierKind::Checkpoint(take(pending_non_checkpoint_barriers)),
+                        },
+                        new_actors,
+                        mutation,
                     }]
                     .into_iter()
-                    .chain(pending_commands.drain(..).map(|command_ctx| {
+                    .chain(pending_upstream_barriers.drain(..).map(|barrier_info| {
                         CreatingJobInjectBarrierInfo {
-                            curr_epoch: command_ctx.curr_epoch.clone(),
-                            prev_epoch: command_ctx.prev_epoch.clone(),
-                            kind: command_ctx.kind.clone(),
+                            barrier_info,
                             new_actors: None,
                             mutation: None,
                         }
                     }))
                     .collect();
 
-                let graph_info = take(graph_info);
-                Some((barriers_to_inject, Some(graph_info)))
-            } else {
+                    *self = CreatingStreamingJobStatus::ConsumingLogStore {
+                        log_store_progress_tracker: CreateMviewLogStoreProgressTracker::new(
+                            snapshot_backfill_actors.iter().cloned(),
+                            barriers_to_inject.len(),
+                        ),
+                    };
+                    Some(barriers_to_inject)
+                } else {
+                    None
+                }
+            }
+            CreatingStreamingJobStatus::ConsumingLogStore {
+                log_store_progress_tracker,
+                ..
+            } => {
+                log_store_progress_tracker.update(create_mview_progress);
+                None
+            }
+            CreatingStreamingJobStatus::Finishing(_) => None,
+        }
+    }
+
+    pub(super) fn on_new_upstream_epoch(
+        &mut self,
+        barrier_info: &BarrierInfo,
+        start_consume_upstream: bool,
+    ) -> Option<CreatingJobInjectBarrierInfo> {
+        match self {
+            CreatingStreamingJobStatus::ConsumingSnapshot {
+                pending_upstream_barriers,
+                prev_epoch_fake_physical_time,
+                pending_non_checkpoint_barriers,
+                initial_barrier_info,
+                ..
+            } => {
+                assert!(
+                    !start_consume_upstream,
+                    "should not start consuming upstream for a job that are consuming snapshot"
+                );
+                pending_upstream_barriers.push(barrier_info.clone());
+                Some(CreatingStreamingJobStatus::new_fake_barrier(
+                    prev_epoch_fake_physical_time,
+                    pending_non_checkpoint_barriers,
+                    initial_barrier_info,
+                    barrier_info.kind.is_checkpoint(),
+                ))
+            }
+            CreatingStreamingJobStatus::ConsumingLogStore { .. } => {
+                let prev_epoch = barrier_info.prev_epoch();
+                if start_consume_upstream {
+                    assert!(barrier_info.kind.is_checkpoint());
+                    *self = CreatingStreamingJobStatus::Finishing(prev_epoch);
+                }
+                Some(CreatingJobInjectBarrierInfo {
+                    barrier_info: barrier_info.clone(),
+                    new_actors: None,
+                    mutation: None,
+                })
+            }
+            CreatingStreamingJobStatus::Finishing { .. } => {
+                assert!(
+                    !start_consume_upstream,
+                    "should not start consuming upstream for a job again"
+                );
+                None
+            }
+        }
+    }
+
+    pub(super) fn new_fake_barrier(
+        prev_epoch_fake_physical_time: &mut u64,
+        pending_non_checkpoint_barriers: &mut Vec<u64>,
+        initial_barrier_info: &mut Option<(HashMap<WorkerId, Vec<StreamActor>>, Mutation)>,
+        is_checkpoint: bool,
+    ) -> CreatingJobInjectBarrierInfo {
+        {
+            {
                 let prev_epoch =
                     TracedEpoch::new(Epoch::from_physical_time(*prev_epoch_fake_physical_time));
                 *prev_epoch_fake_physical_time += 1;
@@ -155,19 +274,20 @@ impl CreatingStreamingJobStatus {
                     } else {
                         Default::default()
                     };
-                Some((
-                    vec![CreatingJobInjectBarrierInfo {
-                        curr_epoch,
+                CreatingJobInjectBarrierInfo {
+                    barrier_info: BarrierInfo {
                         prev_epoch,
+                        curr_epoch,
                         kind,
-                        new_actors,
-                        mutation,
-                    }],
-                    None,
-                ))
+                    },
+                    new_actors,
+                    mutation,
+                }
             }
-        } else {
-            None
         }
     }
+
+    pub(super) fn is_finishing(&self) -> bool {
+        matches!(self, Self::Finishing(_))
+    }
 }
diff --git a/src/meta/src/barrier/info.rs b/src/meta/src/barrier/info.rs
index b02240f402ba0..8153b8f2bc400 100644
--- a/src/meta/src/barrier/info.rs
+++ b/src/meta/src/barrier/info.rs
@@ -14,18 +14,30 @@
 
 use std::collections::{HashMap, HashSet};
 
-use risingwave_common::catalog::TableId;
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_meta_model::WorkerId;
-use risingwave_pb::common::WorkerNode;
 use tracing::warn;
 
-use crate::barrier::Command;
+use crate::barrier::{BarrierKind, Command, TracedEpoch};
 use crate::controller::fragment::InflightFragmentInfo;
 use crate::model::{ActorId, FragmentId};
 
+#[derive(Debug, Clone)]
+pub(super) struct BarrierInfo {
+    pub prev_epoch: TracedEpoch,
+    pub curr_epoch: TracedEpoch,
+    pub kind: BarrierKind,
+}
+
+impl BarrierInfo {
+    pub(super) fn prev_epoch(&self) -> u64 {
+        self.prev_epoch.value().0
+    }
+}
+
 #[derive(Debug, Clone)]
 pub(crate) enum CommandFragmentChanges {
-    NewFragment(InflightFragmentInfo),
+    NewFragment(DatabaseId, TableId, InflightFragmentInfo),
     Reschedule {
         new_actors: HashMap<ActorId, WorkerId>,
         to_remove: HashSet<ActorId>,
@@ -39,69 +51,97 @@ pub struct InflightSubscriptionInfo {
     pub mv_depended_subscriptions: HashMap<TableId, HashMap<u32, u64>>,
 }
 
+#[derive(Clone, Debug)]
+pub struct InflightStreamingJobInfo {
+    pub job_id: TableId,
+    pub database_id: DatabaseId,
+    pub fragment_infos: HashMap<FragmentId, InflightFragmentInfo>,
+}
+
+impl InflightStreamingJobInfo {
+    pub fn fragment_infos(&self) -> impl Iterator<Item = &InflightFragmentInfo> + '_ {
+        self.fragment_infos.values()
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct InflightDatabaseInfo {
+    #[expect(dead_code)]
+    database_id: DatabaseId,
+    jobs: HashMap<TableId, InflightStreamingJobInfo>,
+}
+
+impl InflightDatabaseInfo {
+    pub fn fragment_infos(&self) -> impl Iterator<Item = &InflightFragmentInfo> + '_ {
+        self.jobs.values().flat_map(|job| job.fragment_infos())
+    }
+}
+
 /// [`InflightGraphInfo`] resolves the actor info read from meta store for
-/// [`crate::barrier::GlobalBarrierManager`].
+/// [`crate::barrier::GlobalBarrierWorker`].
 #[derive(Default, Clone, Debug)]
 pub struct InflightGraphInfo {
-    /// `node_id` => actors
-    pub actor_map: HashMap<WorkerId, HashSet<ActorId>>,
-
-    /// `actor_id` => `WorkerId`
-    pub actor_location_map: HashMap<ActorId, WorkerId>,
-
-    pub fragment_infos: HashMap<FragmentId, InflightFragmentInfo>,
+    pub databases: HashMap<DatabaseId, InflightDatabaseInfo>,
+    pub fragment_location: HashMap<FragmentId, (DatabaseId, TableId)>,
 }
 
 impl InflightGraphInfo {
     /// Resolve inflight actor info from given nodes and actors that are loaded from meta store. It will be used during recovery to rebuild all streaming actors.
-    pub fn new(fragment_infos: HashMap<FragmentId, InflightFragmentInfo>) -> Self {
-        let actor_map = {
-            let mut map: HashMap<_, HashSet<_>> = HashMap::new();
-            for info in fragment_infos.values() {
-                for (actor_id, worker_id) in &info.actors {
-                    map.entry(*worker_id as WorkerId)
-                        .or_default()
-                        .insert(*actor_id as ActorId);
+    pub fn new(
+        fragment_infos: HashMap<
+            DatabaseId,
+            HashMap<TableId, HashMap<FragmentId, InflightFragmentInfo>>,
+        >,
+    ) -> Self {
+        let mut fragment_location = HashMap::new();
+        let mut databases = HashMap::new();
+        for (database_id, database_fragment_info) in fragment_infos {
+            let mut database = InflightDatabaseInfo {
+                database_id,
+                jobs: Default::default(),
+            };
+            for (job_id, job_fragment_info) in database_fragment_info {
+                for fragment_id in job_fragment_info.keys() {
+                    fragment_location
+                        .try_insert(*fragment_id, (database_id, job_id))
+                        .expect("no duplicate");
+                }
+                if !job_fragment_info.is_empty() {
+                    database.jobs.insert(
+                        job_id,
+                        InflightStreamingJobInfo {
+                            job_id,
+                            database_id,
+                            fragment_infos: job_fragment_info,
+                        },
+                    );
                 }
             }
-            map
-        };
-
-        let actor_location_map = fragment_infos
-            .values()
-            .flat_map(|fragment| {
-                fragment
-                    .actors
-                    .iter()
-                    .map(|(actor_id, worker_id)| (*actor_id as ActorId, *worker_id as WorkerId))
-            })
-            .collect();
-
+            if !database.jobs.is_empty() {
+                databases
+                    .try_insert(database_id, database)
+                    .expect("no duplicate");
+            }
+        }
         Self {
-            actor_map,
-            actor_location_map,
-            fragment_infos,
+            databases,
+            fragment_location,
         }
     }
 
     pub fn is_empty(&self) -> bool {
-        self.fragment_infos.is_empty()
+        self.databases.is_empty()
     }
 
-    /// Update worker nodes snapshot. We need to support incremental updates for it in the future.
-    pub fn on_new_worker_node_map(&self, node_map: &HashMap<WorkerId, WorkerNode>) {
-        for (node_id, actors) in &self.actor_map {
-            if !node_map.contains_key(node_id) {
-                warn!(node_id, ?actors, "node with running actors is deleted");
-            }
-        }
-    }
-
-    pub(crate) fn extend(&mut self, other: Self) {
+    pub(crate) fn extend(&mut self, job: InflightStreamingJobInfo) {
         self.apply_add(
-            other.fragment_infos.into_iter().map(|(fragment_id, info)| {
-                (fragment_id, CommandFragmentChanges::NewFragment(info))
+            job.fragment_infos.into_iter().map(|(fragment_id, info)| {
+                (
+                    fragment_id,
+                    CommandFragmentChanges::NewFragment(job.database_id, job.job_id, info),
+                )
             }),
+            job.database_id,
         )
     }
 
@@ -110,57 +150,69 @@ impl InflightGraphInfo {
     pub(crate) fn pre_apply(
         &mut self,
         fragment_changes: &HashMap<FragmentId, CommandFragmentChanges>,
+        database_id: DatabaseId,
     ) {
         self.apply_add(
             fragment_changes
                 .iter()
                 .map(|(fragment_id, change)| (*fragment_id, change.clone())),
+            database_id,
         )
     }
 
     fn apply_add(
         &mut self,
         fragment_changes: impl Iterator<Item = (FragmentId, CommandFragmentChanges)>,
+        passed_database_id: DatabaseId,
     ) {
         {
-            let mut to_add = HashMap::new();
             for (fragment_id, change) in fragment_changes {
                 match change {
-                    CommandFragmentChanges::NewFragment(info) => {
-                        for (actor_id, node_id) in &info.actors {
-                            assert!(to_add.insert(*actor_id, *node_id).is_none());
-                        }
-                        assert!(self.fragment_infos.insert(fragment_id, info).is_none());
+                    CommandFragmentChanges::NewFragment(database_id, job_id, info) => {
+                        assert_eq!(passed_database_id, database_id);
+                        let fragment_infos = self
+                            .databases
+                            .entry(database_id)
+                            .or_insert_with(|| InflightDatabaseInfo {
+                                database_id,
+                                jobs: Default::default(),
+                            })
+                            .jobs
+                            .entry(job_id)
+                            .or_insert_with(|| InflightStreamingJobInfo {
+                                job_id,
+                                database_id,
+                                fragment_infos: Default::default(),
+                            });
+                        fragment_infos
+                            .fragment_infos
+                            .try_insert(fragment_id, info)
+                            .expect("non duplicate");
+                        self.fragment_location
+                            .try_insert(fragment_id, (database_id, job_id))
+                            .expect("non duplicate");
                     }
                     CommandFragmentChanges::Reschedule { new_actors, .. } => {
+                        let (database_id, table_id) = self.fragment_location[&fragment_id];
+                        assert_eq!(passed_database_id, database_id);
                         let info = self
+                            .databases
+                            .get_mut(&database_id)
+                            .expect("should exist")
+                            .jobs
+                            .get_mut(&table_id)
+                            .expect("should exist")
                             .fragment_infos
                             .get_mut(&fragment_id)
                             .expect("should exist");
                         let actors = &mut info.actors;
                         for (actor_id, node_id) in &new_actors {
-                            assert!(to_add.insert(*actor_id as _, *node_id as _).is_none());
                             assert!(actors.insert(*actor_id as _, *node_id as _).is_none());
                         }
                     }
                     CommandFragmentChanges::RemoveFragment => {}
                 }
             }
-            for (actor_id, node_id) in to_add {
-                assert!(
-                    self.actor_map
-                        .entry(node_id as _)
-                        .or_default()
-                        .insert(actor_id as _),
-                    "duplicate actor in command changes"
-                );
-                assert!(
-                    self.actor_location_map
-                        .insert(actor_id as _, node_id as _)
-                        .is_none(),
-                    "duplicate actor in command changes"
-                );
-            }
         }
     }
 }
@@ -191,42 +243,46 @@ impl InflightGraphInfo {
     pub(crate) fn post_apply(
         &mut self,
         fragment_changes: &HashMap<FragmentId, CommandFragmentChanges>,
+        passed_database_id: DatabaseId,
     ) {
         {
-            let mut all_to_remove = HashSet::new();
             for (fragment_id, changes) in fragment_changes {
                 match changes {
-                    CommandFragmentChanges::NewFragment(_) => {}
+                    CommandFragmentChanges::NewFragment(_, _, _) => {}
                     CommandFragmentChanges::Reschedule { to_remove, .. } => {
+                        let (database_id, job_id) = self.fragment_location[fragment_id];
+                        assert_eq!(passed_database_id, database_id);
                         let info = self
+                            .databases
+                            .get_mut(&database_id)
+                            .expect("should exist")
+                            .jobs
+                            .get_mut(&job_id)
+                            .expect("should exist")
                             .fragment_infos
                             .get_mut(fragment_id)
                             .expect("should exist");
                         for actor_id in to_remove {
-                            assert!(all_to_remove.insert(*actor_id));
                             assert!(info.actors.remove(&(*actor_id as _)).is_some());
                         }
                     }
                     CommandFragmentChanges::RemoveFragment => {
-                        let info = self
-                            .fragment_infos
+                        let (database_id, job_id) = self.fragment_location[fragment_id];
+                        assert_eq!(passed_database_id, database_id);
+                        let database = self.databases.get_mut(&database_id).expect("should exist");
+                        let job = database.jobs.get_mut(&job_id).expect("should exist");
+                        job.fragment_infos
                             .remove(fragment_id)
                             .expect("should exist");
-                        for (actor_id, _) in info.actors {
-                            assert!(all_to_remove.insert(actor_id as _));
+                        if job.fragment_infos.is_empty() {
+                            database.jobs.remove(&job_id).expect("should exist");
+                            if database.jobs.is_empty() {
+                                self.databases.remove(&database_id).expect("should exist");
+                            }
                         }
                     }
                 }
             }
-            for actor_id in all_to_remove {
-                let node_id = self
-                    .actor_location_map
-                    .remove(&actor_id)
-                    .expect("actor not found");
-                let actor_ids = self.actor_map.get_mut(&node_id).expect("node not found");
-                assert!(actor_ids.remove(&actor_id), "actor not found");
-            }
-            self.actor_map.retain(|_, actor_ids| !actor_ids.is_empty());
         }
     }
 }
@@ -255,24 +311,25 @@ impl InflightSubscriptionInfo {
     }
 }
 
-impl InflightGraphInfo {
+impl InflightFragmentInfo {
     /// Returns actor list to collect in the target worker node.
-    pub fn actor_ids_to_collect(&self, node_id: WorkerId) -> impl Iterator<Item = ActorId> + '_ {
-        self.fragment_infos.values().flat_map(move |info| {
-            info.actors
-                .iter()
-                .filter_map(move |(actor_id, actor_node_id)| {
-                    if (*actor_node_id as WorkerId) == node_id {
-                        Some(*actor_id as _)
-                    } else {
-                        None
-                    }
-                })
-        })
+    pub(crate) fn actor_ids_to_collect(
+        infos: impl IntoIterator<Item = &Self>,
+    ) -> HashMap<WorkerId, HashSet<ActorId>> {
+        let mut ret: HashMap<_, HashSet<_>> = HashMap::new();
+        for (actor_id, worker_id) in infos.into_iter().flat_map(|info| info.actors.iter()) {
+            assert!(ret
+                .entry(*worker_id as WorkerId)
+                .or_default()
+                .insert(*actor_id as _))
+        }
+        ret
     }
 
-    pub fn existing_table_ids(&self) -> impl Iterator<Item = TableId> + '_ {
-        self.fragment_infos.values().flat_map(|info| {
+    pub fn existing_table_ids<'a>(
+        infos: impl IntoIterator<Item = &'a Self> + 'a,
+    ) -> impl Iterator<Item = TableId> + 'a {
+        infos.into_iter().flat_map(|info| {
             info.state_table_ids
                 .iter()
                 .cloned()
@@ -280,11 +337,32 @@ impl InflightGraphInfo {
         })
     }
 
-    pub fn worker_ids(&self) -> impl Iterator<Item = WorkerId> + '_ {
-        self.actor_map.keys().cloned()
+    pub fn contains_worker(infos: impl IntoIterator<Item = &Self>, worker_id: WorkerId) -> bool {
+        infos.into_iter().any(|fragment| {
+            fragment
+                .actors
+                .values()
+                .any(|location| (*location as WorkerId) == worker_id)
+        })
     }
+}
 
+impl InflightGraphInfo {
     pub fn contains_worker(&self, worker_id: WorkerId) -> bool {
-        self.actor_map.contains_key(&worker_id)
+        InflightFragmentInfo::contains_worker(self.fragment_infos(), worker_id)
+    }
+
+    pub fn fragment_infos(&self) -> impl Iterator<Item = &InflightFragmentInfo> + '_ {
+        self.databases
+            .values()
+            .flat_map(|database_info| database_info.fragment_infos())
+    }
+
+    pub fn existing_table_ids(&self) -> impl Iterator<Item = TableId> + '_ {
+        InflightFragmentInfo::existing_table_ids(
+            self.databases
+                .values()
+                .flat_map(|database| database.fragment_infos()),
+        )
     }
 }
diff --git a/src/meta/src/barrier/mod.rs b/src/meta/src/barrier/mod.rs
index 119aed6f0158d..7b9180bb154cc 100644
--- a/src/meta/src/barrier/mod.rs
+++ b/src/meta/src/barrier/mod.rs
@@ -25,24 +25,29 @@ use fail::fail_point;
 use futures::future::try_join_all;
 use itertools::Itertools;
 use prometheus::HistogramTimer;
-use risingwave_common::catalog::TableId;
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::system_param::PAUSE_ON_NEXT_BOOTSTRAP_KEY;
 use risingwave_common::{bail, must_match};
+use risingwave_connector::source::SplitImpl;
 use risingwave_hummock_sdk::change_log::build_table_change_log_delta;
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::from_prost_table_stats_map;
 use risingwave_hummock_sdk::table_watermark::{
     merge_multiple_new_table_watermarks, TableWatermarks,
 };
-use risingwave_hummock_sdk::{HummockSstableObjectId, LocalSstableInfo};
+use risingwave_hummock_sdk::{HummockSstableObjectId, HummockVersionId, LocalSstableInfo};
 use risingwave_meta_model::WorkerId;
+use risingwave_pb::common::WorkerNode;
 use risingwave_pb::ddl_service::DdlProgress;
 use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::meta::{PausedReason, PbRecoveryStatus};
+use risingwave_pb::stream_plan::StreamActor;
 use risingwave_pb::stream_service::barrier_complete_response::CreateMviewProgress;
 use risingwave_pb::stream_service::BarrierCompleteResponse;
+use risingwave_rpc_client::StreamingControlHandle;
 use thiserror_ext::AsReport;
+use tokio::sync::mpsc::unbounded_channel;
 use tokio::sync::oneshot::{Receiver, Sender};
 use tokio::sync::{mpsc, oneshot};
 use tokio::task::JoinHandle;
@@ -50,12 +55,12 @@ use tracing::{debug, error, info, warn, Instrument};
 
 use self::command::CommandContext;
 use self::notifier::Notifier;
-use crate::barrier::creating_job::CreatingStreamingJobControl;
-use crate::barrier::info::InflightGraphInfo;
+use crate::barrier::creating_job::{CompleteJobType, CreatingStreamingJobControl};
+use crate::barrier::info::{BarrierInfo, InflightGraphInfo, InflightStreamingJobInfo};
 use crate::barrier::progress::{CreateMviewProgressTracker, TrackingCommand, TrackingJob};
 use crate::barrier::rpc::{merge_node_rpc_errors, ControlStreamManager};
-use crate::barrier::schedule::ScheduledBarriers;
-use crate::barrier::state::BarrierManagerState;
+use crate::barrier::schedule::{PeriodicBarriers, ScheduledBarriers};
+use crate::barrier::state::BarrierWorkerState;
 use crate::error::MetaErrorInner;
 use crate::hummock::{CommitEpochInfo, HummockManagerRef, NewTableFragmentInfo};
 use crate::manager::sink_coordination::SinkCoordinatorManager;
@@ -63,7 +68,8 @@ use crate::manager::{
     ActiveStreamingWorkerChange, ActiveStreamingWorkerNodes, LocalNotification, MetaSrvEnv,
     MetadataManager,
 };
-use crate::rpc::metrics::MetaMetrics;
+use crate::model::{ActorId, TableFragments};
+use crate::rpc::metrics::GLOBAL_META_METRICS;
 use crate::stream::{ScaleControllerRef, SourceManagerRef};
 use crate::{MetaError, MetaResult};
 
@@ -123,14 +129,19 @@ enum BarrierManagerStatus {
     Running,
 }
 
+struct NewBarrier {
+    command: Option<(DatabaseId, Command, Vec<Notifier>)>,
+    span: tracing::Span,
+    /// Choose a different barrier(checkpoint == true) according to it
+    checkpoint: bool,
+}
+
 /// Scheduled command with its notifiers.
 struct Scheduled {
+    database_id: DatabaseId,
     command: Command,
     notifiers: Vec<Notifier>,
-    send_latency_timer: HistogramTimer,
     span: tracing::Span,
-    /// Choose a different barrier(checkpoint == true) according to it
-    checkpoint: bool,
 }
 
 impl From<&BarrierManagerStatus> for PbRecoveryStatus {
@@ -146,15 +157,14 @@ impl From<&BarrierManagerStatus> for PbRecoveryStatus {
     }
 }
 
-pub enum BarrierManagerRequest {
+pub(crate) enum BarrierManagerRequest {
     GetDdlProgress(Sender<HashMap<u32, DdlProgress>>),
 }
 
-#[derive(Clone)]
-pub struct GlobalBarrierManagerContext {
-    status: Arc<ArcSwap<BarrierManagerStatus>>,
+struct GlobalBarrierWorkerContextImpl {
+    scheduled_barriers: ScheduledBarriers,
 
-    request_tx: mpsc::UnboundedSender<BarrierManagerRequest>,
+    status: Arc<ArcSwap<BarrierManagerStatus>>,
 
     metadata_manager: MetadataManager,
 
@@ -166,42 +176,174 @@ pub struct GlobalBarrierManagerContext {
 
     sink_manager: SinkCoordinatorManager,
 
-    pub(super) metrics: Arc<MetaMetrics>,
-
     env: MetaSrvEnv,
 }
 
-/// [`crate::barrier::GlobalBarrierManager`] sends barriers to all registered compute nodes and
+impl GlobalBarrierManager {
+    pub async fn start(
+        scheduled_barriers: schedule::ScheduledBarriers,
+        env: MetaSrvEnv,
+        metadata_manager: MetadataManager,
+        hummock_manager: HummockManagerRef,
+        source_manager: SourceManagerRef,
+        sink_manager: SinkCoordinatorManager,
+        scale_controller: ScaleControllerRef,
+    ) -> (Arc<Self>, JoinHandle<()>, oneshot::Sender<()>) {
+        let (request_tx, request_rx) = unbounded_channel();
+        let barrier_worker = GlobalBarrierWorker::new(
+            scheduled_barriers,
+            env,
+            metadata_manager,
+            hummock_manager,
+            source_manager,
+            sink_manager,
+            scale_controller,
+            request_rx,
+        )
+        .await;
+        let manager = Self {
+            status: barrier_worker.context.status.clone(),
+            hummock_manager: barrier_worker.context.hummock_manager.clone(),
+            request_tx,
+            metadata_manager: barrier_worker.context.metadata_manager.clone(),
+        };
+        let (join_handle, shutdown_tx) = barrier_worker.start();
+        (Arc::new(manager), join_handle, shutdown_tx)
+    }
+}
+
+trait GlobalBarrierWorkerContext: Send + Sync + 'static {
+    fn commit_epoch(
+        &self,
+        commit_info: CommitEpochInfo,
+    ) -> impl Future<Output = MetaResult<HummockVersionStats>> + Send + '_;
+
+    async fn next_scheduled(&self) -> Scheduled;
+    fn abort_and_mark_blocked(&self, recovery_reason: RecoveryReason);
+    fn mark_ready(&self);
+
+    fn post_collect_command<'a>(
+        &'a self,
+        command: &'a CommandContext,
+    ) -> impl Future<Output = MetaResult<()>> + Send + 'a;
+
+    async fn notify_creating_job_failed(&self, err: &MetaError);
+
+    fn finish_creating_job(
+        &self,
+        job: TrackingJob,
+    ) -> impl Future<Output = MetaResult<()>> + Send + '_;
+
+    async fn new_control_stream(
+        &self,
+        node: &WorkerNode,
+        mv_depended_subscriptions: &HashMap<TableId, HashMap<u32, u64>>,
+    ) -> MetaResult<StreamingControlHandle>;
+
+    async fn reload_runtime_info(
+        &self,
+    ) -> MetaResult<(
+        ActiveStreamingWorkerNodes,
+        InflightGraphInfo,
+        InflightSubscriptionInfo,
+        Option<TracedEpoch>,
+        HashMap<WorkerId, Vec<StreamActor>>,
+        HashMap<ActorId, Vec<SplitImpl>>,
+        HashMap<TableId, (String, TableFragments)>,
+        HummockVersionStats,
+    )>;
+}
+
+impl GlobalBarrierWorkerContext for GlobalBarrierWorkerContextImpl {
+    async fn commit_epoch(&self, commit_info: CommitEpochInfo) -> MetaResult<HummockVersionStats> {
+        self.hummock_manager.commit_epoch(commit_info).await?;
+        Ok(self.hummock_manager.get_version_stats().await)
+    }
+
+    async fn next_scheduled(&self) -> Scheduled {
+        self.scheduled_barriers.next_scheduled().await
+    }
+
+    fn abort_and_mark_blocked(&self, recovery_reason: RecoveryReason) {
+        self.set_status(BarrierManagerStatus::Recovering(recovery_reason));
+
+        // Mark blocked and abort buffered schedules, they might be dirty already.
+        self.scheduled_barriers
+            .abort_and_mark_blocked("cluster is under recovering");
+    }
+
+    fn mark_ready(&self) {
+        self.scheduled_barriers.mark_ready();
+        self.set_status(BarrierManagerStatus::Running);
+    }
+
+    async fn post_collect_command<'a>(&'a self, command: &'a CommandContext) -> MetaResult<()> {
+        command.post_collect(self).await
+    }
+
+    async fn notify_creating_job_failed(&self, err: &MetaError) {
+        self.metadata_manager.notify_finish_failed(err).await
+    }
+
+    async fn finish_creating_job(&self, job: TrackingJob) -> MetaResult<()> {
+        job.finish(&self.metadata_manager).await
+    }
+
+    async fn new_control_stream(
+        &self,
+        node: &WorkerNode,
+        mv_depended_subscriptions: &HashMap<TableId, HashMap<u32, u64>>,
+    ) -> MetaResult<StreamingControlHandle> {
+        self.new_control_stream_impl(node, mv_depended_subscriptions)
+            .await
+    }
+
+    async fn reload_runtime_info(
+        &self,
+    ) -> MetaResult<(
+        ActiveStreamingWorkerNodes,
+        InflightGraphInfo,
+        InflightSubscriptionInfo,
+        Option<TracedEpoch>,
+        HashMap<WorkerId, Vec<StreamActor>>,
+        HashMap<ActorId, Vec<SplitImpl>>,
+        HashMap<TableId, (String, TableFragments)>,
+        HummockVersionStats,
+    )> {
+        self.reload_runtime_info_impl().await
+    }
+}
+
+/// [`crate::barrier::GlobalBarrierWorker`] sends barriers to all registered compute nodes and
 /// collect them, with monotonic increasing epoch numbers. On compute nodes, `LocalBarrierManager`
 /// in `risingwave_stream` crate will serve these requests and dispatch them to source actors.
 ///
 /// Configuration change in our system is achieved by the mutation in the barrier. Thus,
-/// [`crate::barrier::GlobalBarrierManager`] provides a set of interfaces like a state machine,
+/// [`crate::barrier::GlobalBarrierWorker`] provides a set of interfaces like a state machine,
 /// accepting [`Command`] that carries info to build `Mutation`. To keep the consistency between
 /// barrier manager and meta store, some actions like "drop materialized view" or "create mv on mv"
 /// must be done in barrier manager transactional using [`Command`].
-pub struct GlobalBarrierManager {
+struct GlobalBarrierWorker<C> {
     /// Enable recovery or not when failover.
     enable_recovery: bool,
 
     /// The queue of scheduled barriers.
-    scheduled_barriers: schedule::ScheduledBarriers,
+    periodic_barriers: PeriodicBarriers,
 
     /// The max barrier nums in flight
     in_flight_barrier_nums: usize,
 
-    context: GlobalBarrierManagerContext,
+    context: Arc<C>,
 
     env: MetaSrvEnv,
 
-    state: BarrierManagerState,
-
     checkpoint_control: CheckpointControl,
 
-    request_rx: mpsc::UnboundedReceiver<BarrierManagerRequest>,
+    /// Command that has been collected but is still completing.
+    /// The join handle of the completing future is stored.
+    completing_task: CompletingTask,
 
-    /// The `prev_epoch` of pending non checkpoint barriers
-    pending_non_checkpoint_barriers: Vec<u64>,
+    request_rx: mpsc::UnboundedReceiver<BarrierManagerRequest>,
 
     active_streaming_nodes: ActiveStreamingWorkerNodes,
 
@@ -210,64 +352,62 @@ pub struct GlobalBarrierManager {
 
 /// Controls the concurrent execution of commands.
 struct CheckpointControl {
+    state: BarrierWorkerState,
+
     /// Save the state and message of barrier in order.
     /// Key is the `prev_epoch`.
     command_ctx_queue: BTreeMap<u64, EpochNode>,
+    /// The barrier that are completing.
+    /// Some((`prev_epoch`, `should_pause_inject_barrier`))
+    completing_barrier: Option<(u64, bool)>,
 
     creating_streaming_job_controls: HashMap<TableId, CreatingStreamingJobControl>,
 
-    /// Command that has been collected but is still completing.
-    /// The join handle of the completing future is stored.
-    completing_task: CompletingTask,
-
     hummock_version_stats: HummockVersionStats,
 
     create_mview_tracker: CreateMviewProgressTracker,
-
-    context: GlobalBarrierManagerContext,
 }
 
 impl CheckpointControl {
-    async fn new(
-        context: GlobalBarrierManagerContext,
+    fn new(
         create_mview_tracker: CreateMviewProgressTracker,
+        state: BarrierWorkerState,
+        hummock_version_stats: HummockVersionStats,
     ) -> Self {
         Self {
+            state,
             command_ctx_queue: Default::default(),
+            completing_barrier: None,
             creating_streaming_job_controls: Default::default(),
-            completing_task: CompletingTask::None,
-            hummock_version_stats: context.hummock_manager.get_version_stats().await,
+            hummock_version_stats,
             create_mview_tracker,
-            context,
         }
     }
 
     fn total_command_num(&self) -> usize {
         self.command_ctx_queue.len()
-            + match &self.completing_task {
-                CompletingTask::Completing {
-                    command_ctx: Some(_),
-                    ..
-                } => 1,
-                _ => 0,
+            + match &self.completing_barrier {
+                Some(_) => 1,
+                None => 0,
             }
     }
 
     /// Update the metrics of barrier nums.
     fn update_barrier_nums_metrics(&self) {
-        self.context.metrics.in_flight_barrier_nums.set(
+        GLOBAL_META_METRICS.in_flight_barrier_nums.set(
             self.command_ctx_queue
                 .values()
                 .filter(|x| x.state.is_inflight())
                 .count() as i64,
         );
-        self.context
-            .metrics
+        GLOBAL_META_METRICS
             .all_barrier_nums
             .set(self.total_command_num() as i64);
     }
 
-    fn jobs_to_merge(&self) -> Option<HashMap<TableId, (SnapshotBackfillInfo, InflightGraphInfo)>> {
+    fn jobs_to_merge(
+        &self,
+    ) -> Option<HashMap<TableId, (SnapshotBackfillInfo, InflightStreamingJobInfo)>> {
         let mut table_ids_to_merge = HashMap::new();
 
         for (table_id, creating_streaming_job) in &self.creating_streaming_job_controls {
@@ -291,45 +431,27 @@ impl CheckpointControl {
     /// Enqueue a barrier command
     fn enqueue_command(
         &mut self,
-        command_ctx: Arc<CommandContext>,
+        command_ctx: CommandContext,
         notifiers: Vec<Notifier>,
         node_to_collect: HashSet<WorkerId>,
-        jobs_to_wait: HashSet<TableId>,
+        creating_jobs_to_wait: HashSet<TableId>,
     ) {
-        let timer = self.context.metrics.barrier_latency.start_timer();
+        let timer = GLOBAL_META_METRICS.barrier_latency.start_timer();
 
         if let Some((_, node)) = self.command_ctx_queue.last_key_value() {
             assert_eq!(
-                command_ctx.prev_epoch.value(),
-                node.command_ctx.curr_epoch.value()
+                command_ctx.barrier_info.prev_epoch.value(),
+                node.command_ctx.barrier_info.curr_epoch.value()
             );
         }
 
         tracing::trace!(
-            prev_epoch = command_ctx.prev_epoch.value().0,
-            ?jobs_to_wait,
+            prev_epoch = command_ctx.barrier_info.prev_epoch(),
+            ?creating_jobs_to_wait,
             "enqueue command"
         );
-        let creating_jobs_to_wait = jobs_to_wait
-            .into_iter()
-            .map(|table_id| {
-                (
-                    table_id,
-                    if node_to_collect.is_empty() {
-                        Some(
-                            self.creating_streaming_job_controls
-                                .get(&table_id)
-                                .expect("should exist")
-                                .start_wait_progress_timer(),
-                        )
-                    } else {
-                        None
-                    },
-                )
-            })
-            .collect();
         self.command_ctx_queue.insert(
-            command_ctx.prev_epoch.value().0,
+            command_ctx.barrier_info.prev_epoch(),
             EpochNode {
                 enqueue_time: timer,
                 state: BarrierEpochState {
@@ -346,7 +468,11 @@ impl CheckpointControl {
 
     /// Change the state of this `prev_epoch` to `Completed`. Return continuous nodes
     /// with `Completed` starting from first node [`Completed`..`InFlight`) and remove them.
-    fn barrier_collected(&mut self, resp: BarrierCompleteResponse) {
+    fn barrier_collected(
+        &mut self,
+        resp: BarrierCompleteResponse,
+        control_stream_manager: &mut ControlStreamManager,
+    ) -> MetaResult<()> {
         let worker_id = resp.worker_id;
         let prev_epoch = resp.epoch;
         tracing::trace!(
@@ -358,19 +484,6 @@ impl CheckpointControl {
         if resp.partial_graph_id == u32::MAX {
             if let Some(node) = self.command_ctx_queue.get_mut(&prev_epoch) {
                 assert!(node.state.node_to_collect.remove(&(worker_id as _)));
-                if node.state.node_to_collect.is_empty() {
-                    node.state
-                        .creating_jobs_to_wait
-                        .iter_mut()
-                        .for_each(|(table_id, timer)| {
-                            *timer = Some(
-                                self.creating_streaming_job_controls
-                                    .get(table_id)
-                                    .expect("should exist")
-                                    .start_wait_progress_timer(),
-                            );
-                        });
-                }
                 node.state.resps.push(resp);
             } else {
                 panic!(
@@ -383,8 +496,9 @@ impl CheckpointControl {
             self.creating_streaming_job_controls
                 .get_mut(&creating_table_id)
                 .expect("should exist")
-                .collect(prev_epoch, worker_id as _, resp);
+                .collect(prev_epoch, worker_id as _, resp, control_stream_manager)?;
         }
+        Ok(())
     }
 
     /// Pause inject barrier until True.
@@ -400,37 +514,50 @@ impl CheckpointControl {
         let should_pause = self
             .command_ctx_queue
             .last_key_value()
-            .map(|(_, x)| &x.command_ctx)
-            .or(match &self.completing_task {
-                CompletingTask::None | CompletingTask::Err(_) => None,
-                CompletingTask::Completing { command_ctx, .. } => command_ctx.as_ref(),
+            .and_then(|(_, x)| {
+                x.command_ctx
+                    .command
+                    .as_ref()
+                    .map(Command::should_pause_inject_barrier)
             })
-            .map(|command_ctx| command_ctx.command.should_pause_inject_barrier())
+            .or(self
+                .completing_barrier
+                .map(|(_, should_pause)| should_pause))
             .unwrap_or(false);
         debug_assert_eq!(
             self.command_ctx_queue
                 .values()
-                .map(|node| &node.command_ctx)
+                .filter_map(|node| {
+                    node.command_ctx
+                        .command
+                        .as_ref()
+                        .map(Command::should_pause_inject_barrier)
+                })
                 .chain(
-                    match &self.completing_task {
-                        CompletingTask::None | CompletingTask::Err(_) => None,
-                        CompletingTask::Completing { command_ctx, .. } => command_ctx.as_ref(),
-                    }
-                    .into_iter()
+                    self.completing_barrier
+                        .map(|(_, should_pause)| should_pause)
+                        .into_iter()
                 )
-                .any(|command_ctx| command_ctx.command.should_pause_inject_barrier()),
+                .any(|should_pause| should_pause),
             should_pause
         );
 
         in_flight_not_full && !should_pause
     }
+}
 
+impl<C: GlobalBarrierWorkerContext> GlobalBarrierWorker<C> {
     /// We need to make sure there are no changes when doing recovery
     pub async fn clear_on_err(&mut self, err: &MetaError) {
         // join spawned completing command to finish no matter it succeeds or not.
         let is_err = match replace(&mut self.completing_task, CompletingTask::None) {
             CompletingTask::None => false,
-            CompletingTask::Completing { join_handle, .. } => {
+            CompletingTask::Completing {
+                join_handle,
+                command_prev_epoch,
+                creating_job_epochs,
+                ..
+            } => {
                 info!("waiting for completing command to finish in recovery");
                 match join_handle.await {
                     Err(e) => {
@@ -441,41 +568,69 @@ impl CheckpointControl {
                         warn!(err = ?e.as_report(), "failed to complete barrier during clear");
                         true
                     }
-                    Ok(Ok(_)) => false,
+                    Ok(Ok(hummock_version_stats)) => {
+                        self.checkpoint_control
+                            .ack_completed(BarrierCompleteOutput {
+                                command_prev_epoch,
+                                creating_job_epochs,
+                                hummock_version_stats,
+                            });
+                        false
+                    }
                 }
             }
             CompletingTask::Err(_) => true,
         };
         if !is_err {
             // continue to finish the pending collected barrier.
-            while let Some(task) = self.next_complete_barrier_task(None) {
-                if let Err(e) = self.context.clone().complete_barrier(task).await {
-                    error!(
-                        err = ?e.as_report(),
-                        "failed to complete barrier during recovery"
-                    );
-                    break;
-                } else {
-                    info!("succeed to complete barrier during recovery")
+            while let Some(task) = self.checkpoint_control.next_complete_barrier_task(None) {
+                let (command_prev_epoch, creating_job_epochs) = (
+                    task.command_context
+                        .as_ref()
+                        .map(|(command, _)| command.barrier_info.prev_epoch()),
+                    task.creating_job_epochs.clone(),
+                );
+                match task
+                    .complete_barrier(&*self.context, self.env.clone())
+                    .await
+                {
+                    Ok(hummock_version_stats) => {
+                        self.checkpoint_control
+                            .ack_completed(BarrierCompleteOutput {
+                                command_prev_epoch,
+                                creating_job_epochs,
+                                hummock_version_stats,
+                            });
+                    }
+                    Err(e) => {
+                        error!(
+                            err = ?e.as_report(),
+                            "failed to complete barrier during recovery"
+                        );
+                        break;
+                    }
                 }
             }
         }
-        for (_, node) in take(&mut self.command_ctx_queue) {
+        for (_, node) in take(&mut self.checkpoint_control.command_ctx_queue) {
             for notifier in node.notifiers {
                 notifier.notify_failed(err.clone());
             }
             node.enqueue_time.observe_duration();
         }
-        self.create_mview_tracker.abort_all();
+        self.checkpoint_control.create_mview_tracker.abort_all();
     }
+}
 
+impl CheckpointControl {
     /// Return the earliest command waiting on the `worker_id`.
-    fn command_wait_collect_from_worker(&self, worker_id: WorkerId) -> Option<&CommandContext> {
+    fn barrier_wait_collect_from_worker(&self, worker_id: WorkerId) -> Option<&BarrierInfo> {
         for epoch_node in self.command_ctx_queue.values() {
             if epoch_node.state.node_to_collect.contains(&worker_id) {
-                return Some(&epoch_node.command_ctx);
+                return Some(&epoch_node.command_ctx.barrier_info);
             }
         }
+        // TODO: include barrier in creating jobs
         None
     }
 }
@@ -487,8 +642,9 @@ struct EpochNode {
 
     /// Whether this barrier is in-flight or completed.
     state: BarrierEpochState,
+
     /// Context of this command to generate barrier and do some post jobs.
-    command_ctx: Arc<CommandContext>,
+    command_ctx: CommandContext,
     /// Notifiers of this barrier.
     notifiers: Vec<Notifier>,
 }
@@ -500,9 +656,9 @@ struct BarrierEpochState {
 
     resps: Vec<BarrierCompleteResponse>,
 
-    creating_jobs_to_wait: HashMap<TableId, Option<HistogramTimer>>,
+    creating_jobs_to_wait: HashSet<TableId>,
 
-    finished_jobs: HashMap<TableId, CreateStreamingJobCommandInfo>,
+    finished_jobs: HashMap<TableId, (CreateStreamingJobCommandInfo, Vec<BarrierCompleteResponse>)>,
 }
 
 impl BarrierEpochState {
@@ -514,8 +670,7 @@ impl BarrierEpochState {
 enum CompletingTask {
     None,
     Completing {
-        command_ctx: Option<Arc<CommandContext>>,
-        table_ids_to_finish: HashSet<TableId>,
+        command_prev_epoch: Option<u64>,
         creating_job_epochs: Vec<(TableId, u64)>,
 
         // The join handle of a spawned task that completes the barrier.
@@ -527,9 +682,8 @@ enum CompletingTask {
     Err(MetaError),
 }
 
-impl GlobalBarrierManager {
-    /// Create a new [`crate::barrier::GlobalBarrierManager`].
-    #[allow(clippy::too_many_arguments)]
+impl GlobalBarrierWorker<GlobalBarrierWorkerContextImpl> {
+    /// Create a new [`crate::barrier::GlobalBarrierWorker`].
     pub async fn new(
         scheduled_barriers: schedule::ScheduledBarriers,
         env: MetaSrvEnv,
@@ -537,63 +691,71 @@ impl GlobalBarrierManager {
         hummock_manager: HummockManagerRef,
         source_manager: SourceManagerRef,
         sink_manager: SinkCoordinatorManager,
-        metrics: Arc<MetaMetrics>,
         scale_controller: ScaleControllerRef,
+        request_rx: mpsc::UnboundedReceiver<BarrierManagerRequest>,
     ) -> Self {
         let enable_recovery = env.opts.enable_recovery;
         let in_flight_barrier_nums = env.opts.in_flight_barrier_nums;
 
-        let initial_invalid_state = BarrierManagerState::new(
+        let initial_invalid_state = BarrierWorkerState::new(
             None,
             InflightGraphInfo::default(),
             InflightSubscriptionInfo::default(),
             None,
         );
 
-        let active_streaming_nodes = ActiveStreamingWorkerNodes::uninitialized();
+        let active_streaming_nodes =
+            ActiveStreamingWorkerNodes::uninitialized(metadata_manager.clone());
 
         let tracker = CreateMviewProgressTracker::default();
 
-        let (request_tx, request_rx) = mpsc::unbounded_channel();
+        let status = Arc::new(ArcSwap::new(Arc::new(BarrierManagerStatus::Starting)));
 
-        let context = GlobalBarrierManagerContext {
-            status: Arc::new(ArcSwap::new(Arc::new(BarrierManagerStatus::Starting))),
-            request_tx,
+        let context = Arc::new(GlobalBarrierWorkerContextImpl {
+            scheduled_barriers,
+            status,
             metadata_manager,
             hummock_manager,
             source_manager,
             scale_controller,
             sink_manager,
-            metrics,
             env: env.clone(),
-        };
+        });
+
+        let control_stream_manager = ControlStreamManager::new(env.clone());
+        let checkpoint_control = CheckpointControl::new(
+            tracker,
+            initial_invalid_state,
+            context.hummock_manager.get_version_stats().await,
+        );
 
-        let control_stream_manager = ControlStreamManager::new(context.clone());
-        let checkpoint_control = CheckpointControl::new(context.clone(), tracker).await;
+        let checkpoint_frequency = env.system_params_reader().await.checkpoint_frequency() as _;
+        let interval =
+            Duration::from_millis(env.system_params_reader().await.barrier_interval_ms() as u64);
+        let periodic_barriers = PeriodicBarriers::new(interval, checkpoint_frequency);
+        tracing::info!(
+            "Starting barrier scheduler with: checkpoint_frequency={:?}",
+            checkpoint_frequency,
+        );
 
         Self {
             enable_recovery,
-            scheduled_barriers,
+            periodic_barriers,
             in_flight_barrier_nums,
             context,
             env,
-            state: initial_invalid_state,
             checkpoint_control,
+            completing_task: CompletingTask::None,
             request_rx,
-            pending_non_checkpoint_barriers: Vec::new(),
             active_streaming_nodes,
             control_stream_manager,
         }
     }
 
-    pub fn context(&self) -> &GlobalBarrierManagerContext {
-        &self.context
-    }
-
-    pub fn start(barrier_manager: GlobalBarrierManager) -> (JoinHandle<()>, Sender<()>) {
+    pub fn start(self) -> (JoinHandle<()>, Sender<()>) {
         let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
         let join_handle = tokio::spawn(async move {
-            barrier_manager.run(shutdown_rx).await;
+            self.run(shutdown_rx).await;
         });
 
         (join_handle, shutdown_tx)
@@ -622,15 +784,9 @@ impl GlobalBarrierManager {
     }
 
     /// Start an infinite loop to take scheduled barriers and send them.
-    async fn run(mut self, mut shutdown_rx: Receiver<()>) {
-        // Initialize the barrier manager.
-        let interval = Duration::from_millis(
-            self.env.system_params_reader().await.barrier_interval_ms() as u64,
-        );
-        self.scheduled_barriers.set_min_interval(interval);
+    async fn run(mut self, shutdown_rx: Receiver<()>) {
         tracing::info!(
-            "Starting barrier manager with: interval={:?}, enable_recovery={}, in_flight_barrier_nums={}",
-            interval,
+            "Starting barrier manager with: enable_recovery={}, in_flight_barrier_nums={}",
             self.enable_recovery,
             self.in_flight_barrier_nums,
         );
@@ -656,8 +812,6 @@ impl GlobalBarrierManager {
             // consistency.
             // Even if there's no actor to recover, we still go through the recovery process to
             // inject the first `Initial` barrier.
-            self.context
-                .set_status(BarrierManagerStatus::Recovering(RecoveryReason::Bootstrap));
             let span = tracing::info_span!("bootstrap_recovery");
             crate::telemetry::report_event(
                 risingwave_pb::telemetry::TelemetryEventStage::Recovery,
@@ -671,11 +825,17 @@ impl GlobalBarrierManager {
             let paused = self.take_pause_on_bootstrap().await.unwrap_or(false);
             let paused_reason = paused.then_some(PausedReason::Manual);
 
-            self.recovery(paused_reason, None).instrument(span).await;
+            self.recovery(paused_reason, None, RecoveryReason::Bootstrap)
+                .instrument(span)
+                .await;
         }
 
-        self.context.set_status(BarrierManagerStatus::Running);
+        self.run_inner(shutdown_rx).await
+    }
+}
 
+impl<C: GlobalBarrierWorkerContext> GlobalBarrierWorker<C> {
+    async fn run_inner(mut self, mut shutdown_rx: Receiver<()>) {
         let (local_notification_tx, mut local_notification_rx) =
             tokio::sync::mpsc::unbounded_channel();
         self.env
@@ -718,57 +878,13 @@ impl GlobalBarrierManager {
                 changed_worker = self.active_streaming_nodes.changed() => {
                     #[cfg(debug_assertions)]
                     {
-                        use risingwave_pb::common::WorkerNode;
-                        match self
-                            .context
-                            .metadata_manager
-                            .list_active_streaming_compute_nodes()
-                            .await
-                        {
-                            Ok(worker_nodes) => {
-                                let ignore_irrelevant_info = |node: &WorkerNode| {
-                                    (
-                                        node.id,
-                                        WorkerNode {
-                                            id: node.id,
-                                            r#type: node.r#type,
-                                            host: node.host.clone(),
-                                            parallelism: node.parallelism,
-                                            property: node.property.clone(),
-                                            resource: node.resource.clone(),
-                                            ..Default::default()
-                                        },
-                                    )
-                                };
-                                let worker_nodes: HashMap<_, _> =
-                                    worker_nodes.iter().map(ignore_irrelevant_info).collect();
-                                let curr_worker_nodes: HashMap<_, _> = self
-                                    .active_streaming_nodes
-                                    .current()
-                                    .values()
-                                    .map(ignore_irrelevant_info)
-                                    .collect();
-                                if worker_nodes != curr_worker_nodes {
-                                    warn!(
-                                        ?worker_nodes,
-                                        ?curr_worker_nodes,
-                                        "different to global snapshot"
-                                    );
-                                }
-                            }
-                            Err(e) => {
-                                warn!(e = ?e.as_report(), "fail to list_active_streaming_compute_nodes to compare with local snapshot");
-                            }
-                        }
+                        self.active_streaming_nodes.validate_change().await;
                     }
 
                     info!(?changed_worker, "worker changed");
 
-                    self.state.inflight_graph_info
-                        .on_new_worker_node_map(self.active_streaming_nodes.current());
-                    self.checkpoint_control.creating_streaming_job_controls.values().for_each(|job| job.on_new_worker_node_map(self.active_streaming_nodes.current()));
                     if let ActiveStreamingWorkerChange::Add(node) | ActiveStreamingWorkerChange::Update(node) = changed_worker {
-                        self.control_stream_manager.add_worker(node, &self.state.inflight_subscription_info).await;
+                        self.control_stream_manager.add_worker(node, &self.checkpoint_control.state.inflight_subscription_info, &*self.context).await;
                     }
                 }
 
@@ -777,8 +893,8 @@ impl GlobalBarrierManager {
                     match notification {
                         // Handle barrier interval and checkpoint frequency changes.
                         LocalNotification::SystemParamsChange(p) => {
-                            self.scheduled_barriers.set_min_interval(Duration::from_millis(p.barrier_interval_ms() as u64));
-                            self.scheduled_barriers
+                            self.periodic_barriers.set_min_interval(Duration::from_millis(p.barrier_interval_ms() as u64));
+                            self.periodic_barriers
                                 .set_checkpoint_frequency(p.checkpoint_frequency() as usize)
                         },
                         // Handle adhoc recovery triggered by user.
@@ -788,21 +904,35 @@ impl GlobalBarrierManager {
                         _ => {}
                     }
                 }
-                (worker_id, resp_result) = self.control_stream_manager.next_complete_barrier_response() => {
-                    match resp_result {
-                        Ok(resp) => {
-                            self.checkpoint_control.barrier_collected(resp);
-
+                complete_result = self
+                    .completing_task
+                    .next_completed_barrier(
+                        &mut self.periodic_barriers,
+                        &mut self.checkpoint_control,
+                        &mut self.control_stream_manager,
+                        &self.context,
+                        &self.env,
+                ) => {
+                    match complete_result {
+                        Ok(output) => {
+                            self.checkpoint_control.ack_completed(output);
                         }
                         Err(e) => {
-                            let failed_command = self.checkpoint_control.command_wait_collect_from_worker(worker_id as _);
-                            if failed_command.is_some()
-                                || self.state.inflight_graph_info.contains_worker(worker_id as _)
-                                || self.checkpoint_control.creating_streaming_job_controls.values().any(|job| job.is_wait_on_worker(worker_id as _)) {
+                            self.failure_recovery(e).await;
+                        }
+                    }
+                },
+                (worker_id, resp_result) = self.control_stream_manager.next_collect_barrier_response() => {
+                    if let  Err(e) = resp_result.and_then(|resp| self.checkpoint_control.barrier_collected(resp, &mut self.control_stream_manager)) {
+                        {
+                            let failed_barrier = self.checkpoint_control.barrier_wait_collect_from_worker(worker_id as _);
+                            if failed_barrier.is_some()
+                                || self.checkpoint_control.state.inflight_graph_info.contains_worker(worker_id as _)
+                                || self.checkpoint_control.creating_streaming_job_controls.values().any(|job| job.is_wait_on_worker(worker_id)) {
                                 let errors = self.control_stream_manager.collect_errors(worker_id, e).await;
                                 let err = merge_node_rpc_errors("get error from control stream", errors);
-                                if let Some(failed_command) = failed_command {
-                                    self.context.report_collect_failure(failed_command, &err);
+                                if let Some(failed_barrier) = failed_barrier {
+                                    self.report_collect_failure(failed_barrier, &err);
                                 }
                                 self.failure_recovery(err).await;
                             } else {
@@ -811,25 +941,11 @@ impl GlobalBarrierManager {
                         }
                     }
                 }
-                complete_result = self.checkpoint_control.next_completed_barrier(&mut self.scheduled_barriers) => {
-                    match complete_result {
-                        Ok(output) => {
-                            if !output.table_ids_to_finish.is_empty() {
-                                self.control_stream_manager.remove_partial_graph(
-                                    output.table_ids_to_finish.iter().map(|table_id| table_id.table_id).collect()
-                                );
-                            }
-                        }
-                        Err(e) => {
-                            self.failure_recovery(e).await;
-                        }
-                    }
-                },
-                scheduled = self.scheduled_barriers.next_barrier(),
+                new_barrier = self.periodic_barriers.next_barrier(&*self.context),
                     if self
                         .checkpoint_control
                         .can_inject_barrier(self.in_flight_barrier_nums) => {
-                    if let Err(e) = self.handle_new_barrier(scheduled) {
+                    if let Err(e) = self.checkpoint_control.handle_new_barrier(new_barrier, &mut self.control_stream_manager, &self.active_streaming_nodes) {
                         self.failure_recovery(e).await;
                     }
                 }
@@ -837,20 +953,31 @@ impl GlobalBarrierManager {
             self.checkpoint_control.update_barrier_nums_metrics();
         }
     }
+}
 
+impl CheckpointControl {
     /// Handle the new barrier from the scheduled queue and inject it.
-    fn handle_new_barrier(&mut self, scheduled: Scheduled) -> MetaResult<()> {
-        let Scheduled {
-            mut command,
-            mut notifiers,
-            send_latency_timer,
+    fn handle_new_barrier(
+        &mut self,
+        new_barrier: NewBarrier,
+        control_stream_manager: &mut ControlStreamManager,
+        active_streaming_nodes: &ActiveStreamingWorkerNodes,
+    ) -> MetaResult<()> {
+        let NewBarrier {
+            command,
             checkpoint,
             span,
-        } = scheduled;
+        } = new_barrier;
+
+        let (mut command, mut notifiers, database_id) =
+            if let Some((database_id, command, notifiers)) = command {
+                (Some(command), notifiers, Some(database_id))
+            } else {
+                (None, vec![], None)
+            };
 
-        if let Some(table_to_cancel) = command.table_to_cancel()
+        if let Some(table_to_cancel) = command.as_ref().and_then(Command::table_to_cancel)
             && self
-                .checkpoint_control
                 .creating_streaming_job_controls
                 .contains_key(&table_to_cancel)
         {
@@ -865,12 +992,8 @@ impl GlobalBarrierManager {
             return Ok(());
         }
 
-        if let Command::RescheduleFragment { .. } = &command {
-            if !self
-                .checkpoint_control
-                .creating_streaming_job_controls
-                .is_empty()
-            {
+        if let Some(Command::RescheduleFragment { .. }) = &command {
+            if !self.creating_streaming_job_controls.is_empty() {
                 warn!("ignore reschedule when creating streaming job with snapshot backfill");
                 for notifier in notifiers {
                     notifier.notify_start_failed(
@@ -884,7 +1007,7 @@ impl GlobalBarrierManager {
             }
         }
 
-        let Some((prev_epoch, curr_epoch)) = self.state.next_epoch_pair(&command) else {
+        let Some(barrier_info) = self.state.next_barrier_info(command.as_ref(), checkpoint) else {
             // skip the command when there is nothing to do with the barrier
             for mut notifier in notifiers {
                 notifier.notify_started();
@@ -894,10 +1017,10 @@ impl GlobalBarrierManager {
         };
 
         // Insert newly added creating job
-        if let Command::CreateStreamingJob {
+        if let Some(Command::CreateStreamingJob {
             job_type: CreateStreamingJobType::SnapshotBackfill(snapshot_backfill_info),
             info,
-        } = &command
+        }) = &command
         {
             if self.state.paused_reason().is_some() {
                 warn!("cannot create streaming job with snapshot backfill when paused");
@@ -910,98 +1033,55 @@ impl GlobalBarrierManager {
                 return Ok(());
             }
             let mutation = command
+                .as_ref()
+                .expect("checked Some")
                 .to_mutation(None)
                 .expect("should have some mutation in `CreateStreamingJob` command");
-            self.checkpoint_control
-                .creating_streaming_job_controls
-                .insert(
-                    info.table_fragments.table_id(),
-                    CreatingStreamingJobControl::new(
-                        info.clone(),
-                        snapshot_backfill_info.clone(),
-                        prev_epoch.value().0,
-                        &self.checkpoint_control.hummock_version_stats,
-                        &self.context.metrics,
-                        mutation,
-                    ),
-                );
-        }
-
-        // may inject fake barrier
-        for creating_job in self
-            .checkpoint_control
-            .creating_streaming_job_controls
-            .values_mut()
-        {
-            creating_job.may_inject_fake_barrier(
-                &mut self.control_stream_manager,
-                prev_epoch.value().0,
-                checkpoint,
-            )?
+            self.creating_streaming_job_controls.insert(
+                info.table_fragments.table_id(),
+                CreatingStreamingJobControl::new(
+                    info.clone(),
+                    snapshot_backfill_info.clone(),
+                    barrier_info.prev_epoch(),
+                    &self.hummock_version_stats,
+                    mutation,
+                ),
+            );
         }
 
-        self.pending_non_checkpoint_barriers
-            .push(prev_epoch.value().0);
-        let kind = if checkpoint {
-            let epochs = take(&mut self.pending_non_checkpoint_barriers);
-            BarrierKind::Checkpoint(epochs)
-        } else {
-            BarrierKind::Barrier
-        };
-
-        tracing::trace!(prev_epoch = prev_epoch.value().0, "inject barrier");
-
         // Collect the jobs to finish
-        if let (BarrierKind::Checkpoint(_), Command::Plain(None)) = (&kind, &command)
-            && let Some(jobs_to_merge) = self.checkpoint_control.jobs_to_merge()
+        if let (BarrierKind::Checkpoint(_), None) = (&barrier_info.kind, &command)
+            && let Some(jobs_to_merge) = self.jobs_to_merge()
         {
-            command = Command::MergeSnapshotBackfillStreamingJobs(jobs_to_merge);
+            command = Some(Command::MergeSnapshotBackfillStreamingJobs(jobs_to_merge));
         }
 
-        let (pre_applied_graph_info, pre_applied_subscription_info) =
-            self.state.apply_command(&command);
-
-        // Tracing related stuff
-        prev_epoch.span().in_scope(|| {
-            tracing::info!(target: "rw_tracing", epoch = curr_epoch.value().0, "new barrier enqueued");
-        });
-        span.record("epoch", curr_epoch.value().0);
-
-        let table_ids_to_commit: HashSet<_> = pre_applied_graph_info.existing_table_ids().collect();
+        let command = command;
 
-        let command_ctx = Arc::new(CommandContext::new(
-            self.active_streaming_nodes.current().clone(),
+        let (
+            pre_applied_graph_info,
             pre_applied_subscription_info,
-            prev_epoch.clone(),
-            curr_epoch.clone(),
             table_ids_to_commit,
-            self.state.paused_reason(),
-            command,
-            kind,
-            self.context.clone(),
-            span,
-        ));
-
-        send_latency_timer.observe_duration();
+            jobs_to_wait,
+            prev_paused_reason,
+        ) = self.state.apply_command(command.as_ref(), database_id);
 
-        let mut jobs_to_wait = HashSet::new();
+        // Tracing related stuff
+        barrier_info.prev_epoch.span().in_scope(|| {
+            tracing::info!(target: "rw_tracing", epoch = barrier_info.curr_epoch.value().0, "new barrier enqueued");
+        });
+        span.record("epoch", barrier_info.curr_epoch.value().0);
 
-        for (table_id, creating_job) in &mut self.checkpoint_control.creating_streaming_job_controls
-        {
-            if let Some(wait_job) =
-                creating_job.on_new_command(&mut self.control_stream_manager, &command_ctx)?
-            {
-                jobs_to_wait.insert(*table_id);
-                if let Some(graph_to_finish) = wait_job {
-                    self.state.inflight_graph_info.extend(graph_to_finish);
-                }
-            }
+        for creating_job in &mut self.creating_streaming_job_controls.values_mut() {
+            creating_job.on_new_command(control_stream_manager, command.as_ref(), &barrier_info)?;
         }
 
-        let node_to_collect = match self.control_stream_manager.inject_command_ctx_barrier(
-            &command_ctx,
+        let node_to_collect = match control_stream_manager.inject_command_ctx_barrier(
+            command.as_ref(),
+            &barrier_info,
+            prev_paused_reason,
             &pre_applied_graph_info,
-            Some(&self.state.inflight_graph_info),
+            &self.state.inflight_graph_info,
         ) {
             Ok(node_to_collect) => node_to_collect,
             Err(err) => {
@@ -1014,32 +1094,30 @@ impl GlobalBarrierManager {
         };
 
         // Notify about the injection.
-        let curr_paused_reason = command_ctx.next_paused_reason();
-
         notifiers.iter_mut().for_each(|n| n.notify_started());
 
-        // Update the paused state after the barrier is injected.
-        self.state.set_paused_reason(curr_paused_reason);
-        // Record the in-flight barrier.
-        self.checkpoint_control.enqueue_command(
-            command_ctx,
-            notifiers,
-            node_to_collect,
-            jobs_to_wait,
+        let command_ctx = CommandContext::new(
+            active_streaming_nodes.current().clone(),
+            barrier_info,
+            pre_applied_subscription_info,
+            table_ids_to_commit.clone(),
+            command,
+            span,
         );
 
+        // Record the in-flight barrier.
+        self.enqueue_command(command_ctx, notifiers, node_to_collect, jobs_to_wait);
+
         Ok(())
     }
+}
 
+impl<C: GlobalBarrierWorkerContext> GlobalBarrierWorker<C> {
+    /// Set barrier manager status.
     async fn failure_recovery(&mut self, err: MetaError) {
-        self.checkpoint_control.clear_on_err(&err).await;
-        self.pending_non_checkpoint_barriers.clear();
+        self.clear_on_err(&err).await;
 
         if self.enable_recovery {
-            self.context
-                .set_status(BarrierManagerStatus::Recovering(RecoveryReason::Failover(
-                    err.clone(),
-                )));
             let span = tracing::info_span!(
                 "failure_recovery",
                 error = %err.as_report(),
@@ -1054,21 +1132,25 @@ impl GlobalBarrierManager {
                 None,
             );
 
+            let reason = RecoveryReason::Failover(err.clone());
+
             // No need to clean dirty tables for barrier recovery,
             // The foreground stream job should cleanup their own tables.
-            self.recovery(None, Some(err)).instrument(span).await;
-            self.context.set_status(BarrierManagerStatus::Running);
+            self.recovery(None, Some(err), reason)
+                .instrument(span)
+                .await;
         } else {
-            panic!("failed to execute barrier: {}", err.as_report());
+            panic!(
+                "a streaming error occurred while recovery is disabled, aborting: {:?}",
+                err.as_report()
+            );
         }
     }
 
     async fn adhoc_recovery(&mut self) {
         let err = MetaErrorInner::AdhocRecovery.into();
-        self.checkpoint_control.clear_on_err(&err).await;
+        self.clear_on_err(&err).await;
 
-        self.context
-            .set_status(BarrierManagerStatus::Recovering(RecoveryReason::Adhoc));
         let span = tracing::info_span!(
             "adhoc_recovery",
             error = %err.as_report(),
@@ -1085,23 +1167,27 @@ impl GlobalBarrierManager {
 
         // No need to clean dirty tables for barrier recovery,
         // The foreground stream job should cleanup their own tables.
-        self.recovery(None, Some(err)).instrument(span).await;
-        self.context.set_status(BarrierManagerStatus::Running);
+        self.recovery(None, Some(err), RecoveryReason::Adhoc)
+            .instrument(span)
+            .await;
     }
 }
 
 #[derive(Default)]
-struct CompleteBarrierTask {
+pub struct CompleteBarrierTask {
     commit_info: CommitEpochInfo,
     finished_jobs: Vec<TrackingJob>,
     notifiers: Vec<Notifier>,
     /// Some((`command_ctx`, `enqueue_time`))
-    command_context: Option<(Arc<CommandContext>, HistogramTimer)>,
-    table_ids_to_finish: HashSet<TableId>,
+    command_context: Option<(CommandContext, HistogramTimer)>,
     creating_job_epochs: Vec<(TableId, u64)>,
 }
 
-impl GlobalBarrierManagerContext {
+impl GlobalBarrierWorkerContextImpl {
+    fn set_status(&self, new_status: BarrierManagerStatus) {
+        self.status.store(Arc::new(new_status));
+    }
+
     fn collect_creating_job_commit_epoch_info(
         commit_info: &mut CommitEpochInfo,
         epoch: u64,
@@ -1127,53 +1213,62 @@ impl GlobalBarrierManagerContext {
         if is_first_time {
             commit_info
                 .new_table_fragment_infos
-                .push(NewTableFragmentInfo::NewCompactionGroup {
+                .push(NewTableFragmentInfo {
                     table_ids: tables_to_commit,
                 });
         };
     }
+}
 
-    async fn complete_barrier(self, task: CompleteBarrierTask) -> MetaResult<HummockVersionStats> {
-        let result: MetaResult<()> = try {
-            let wait_commit_timer = self.metrics.barrier_wait_commit_latency.start_timer();
-            self.hummock_manager.commit_epoch(task.commit_info).await?;
-            if let Some((command_ctx, _)) = &task.command_context {
-                command_ctx.post_collect().await?;
+impl CompleteBarrierTask {
+    async fn complete_barrier(
+        self,
+        context: &impl GlobalBarrierWorkerContext,
+        env: MetaSrvEnv,
+    ) -> MetaResult<HummockVersionStats> {
+        let result: MetaResult<HummockVersionStats> = try {
+            let wait_commit_timer = GLOBAL_META_METRICS
+                .barrier_wait_commit_latency
+                .start_timer();
+            let version_stats = context.commit_epoch(self.commit_info).await?;
+            if let Some((command_ctx, _)) = &self.command_context {
+                context.post_collect_command(command_ctx).await?;
             }
 
             wait_commit_timer.observe_duration();
+            version_stats
         };
 
-        {
-            if let Err(e) = result {
-                for notifier in task.notifiers {
-                    notifier.notify_collection_failed(e.clone());
+        let version_stats = {
+            let version_stats = match result {
+                Ok(version_stats) => version_stats,
+                Err(e) => {
+                    for notifier in self.notifiers {
+                        notifier.notify_collection_failed(e.clone());
+                    }
+                    return Err(e);
                 }
-                return Err(e);
-            }
-            task.notifiers.into_iter().for_each(|notifier| {
+            };
+            self.notifiers.into_iter().for_each(|notifier| {
                 notifier.notify_collected();
             });
             try_join_all(
-                task.finished_jobs
+                self.finished_jobs
                     .into_iter()
-                    .map(|finished_job| finished_job.finish(&self.metadata_manager)),
+                    .map(|finished_job| context.finish_creating_job(finished_job)),
             )
             .await?;
-            if let Some((command_ctx, enqueue_time)) = task.command_context {
+            if let Some((command_ctx, enqueue_time)) = self.command_context {
                 let duration_sec = enqueue_time.stop_and_record();
-                self.report_complete_event(duration_sec, &command_ctx);
-                self.metrics
+                Self::report_complete_event(env, duration_sec, &command_ctx);
+                GLOBAL_META_METRICS
                     .last_committed_barrier_time
-                    .set(command_ctx.curr_epoch.value().as_unix_secs() as i64);
+                    .set(command_ctx.barrier_info.curr_epoch.value().as_unix_secs() as i64);
             }
-        }
-
-        Ok(self.hummock_manager.get_version_stats().await)
-    }
+            version_stats
+        };
 
-    pub fn hummock_manager(&self) -> &HummockManagerRef {
-        &self.hummock_manager
+        Ok(version_stats)
     }
 }
 
@@ -1218,25 +1313,30 @@ impl CreateMviewProgressTracker {
     }
 }
 
-impl GlobalBarrierManagerContext {
-    fn report_complete_event(&self, duration_sec: f64, command_ctx: &CommandContext) {
+impl CompleteBarrierTask {
+    fn report_complete_event(env: MetaSrvEnv, duration_sec: f64, command_ctx: &CommandContext) {
         // Record barrier latency in event log.
         use risingwave_pb::meta::event_log;
         let event = event_log::EventBarrierComplete {
-            prev_epoch: command_ctx.prev_epoch.value().0,
-            cur_epoch: command_ctx.curr_epoch.value().0,
+            prev_epoch: command_ctx.barrier_info.prev_epoch(),
+            cur_epoch: command_ctx.barrier_info.curr_epoch.value().0,
             duration_sec,
-            command: command_ctx.command.to_string(),
-            barrier_kind: command_ctx.kind.as_str_name().to_string(),
+            command: command_ctx
+                .command
+                .as_ref()
+                .map(|command| command.to_string())
+                .unwrap_or_else(|| "barrier".to_string()),
+            barrier_kind: command_ctx.barrier_info.kind.as_str_name().to_string(),
         };
-        self.env
-            .event_log_manager_ref()
+        env.event_log_manager_ref()
             .add_event_logs(vec![event_log::Event::BarrierComplete(event)]);
     }
 }
 
 struct BarrierCompleteOutput {
-    table_ids_to_finish: HashSet<TableId>,
+    command_prev_epoch: Option<u64>,
+    creating_job_epochs: Vec<(TableId, u64)>,
+    hummock_version_stats: HummockVersionStats,
 }
 
 impl CheckpointControl {
@@ -1267,9 +1367,65 @@ impl CheckpointControl {
 
     fn next_complete_barrier_task(
         &mut self,
-        mut scheduled_barriers: Option<&mut ScheduledBarriers>,
+        mut context: Option<(&mut PeriodicBarriers, &mut ControlStreamManager)>,
     ) -> Option<CompleteBarrierTask> {
+        // `Vec::new` is a const fn, and do not have memory allocation, and therefore is lightweight enough
+        let mut creating_jobs_task = vec![];
+        {
+            // `Vec::new` is a const fn, and do not have memory allocation, and therefore is lightweight enough
+            let mut finished_jobs = Vec::new();
+            let min_upstream_inflight_barrier = self
+                .command_ctx_queue
+                .first_key_value()
+                .map(|(epoch, _)| *epoch);
+            for (table_id, job) in &mut self.creating_streaming_job_controls {
+                if let Some((epoch, resps, status)) =
+                    job.start_completing(min_upstream_inflight_barrier)
+                {
+                    let is_first_time = match status {
+                        CompleteJobType::First => true,
+                        CompleteJobType::Normal => false,
+                        CompleteJobType::Finished => {
+                            finished_jobs.push((*table_id, epoch, resps));
+                            continue;
+                        }
+                    };
+                    creating_jobs_task.push((*table_id, epoch, resps, is_first_time));
+                }
+            }
+            if !finished_jobs.is_empty()
+                && let Some((_, control_stream_manager)) = &mut context
+            {
+                control_stream_manager.remove_partial_graph(
+                    finished_jobs
+                        .iter()
+                        .map(|(table_id, _, _)| table_id.table_id)
+                        .collect(),
+                );
+            }
+            for (table_id, epoch, resps) in finished_jobs {
+                let epoch_state = &mut self
+                    .command_ctx_queue
+                    .get_mut(&epoch)
+                    .expect("should exist")
+                    .state;
+                assert!(epoch_state.creating_jobs_to_wait.remove(&table_id));
+                debug!(epoch, ?table_id, "finish creating job");
+                // It's safe to remove the creating job, because on CompleteJobType::Finished,
+                // all previous barriers have been collected and completed.
+                let creating_streaming_job = self
+                    .creating_streaming_job_controls
+                    .remove(&table_id)
+                    .expect("should exist");
+                assert!(creating_streaming_job.is_finished());
+                assert!(epoch_state
+                    .finished_jobs
+                    .insert(table_id, (creating_streaming_job.info, resps))
+                    .is_none());
+            }
+        }
         let mut task = None;
+        assert!(self.completing_barrier.is_none());
         while let Some((_, EpochNode { state, .. })) = self.command_ctx_queue.first_key_value()
             && !state.is_inflight()
         {
@@ -1280,104 +1436,104 @@ impl CheckpointControl {
                 let mut finished_jobs = self
                     .create_mview_tracker
                     .apply_collected_command(&node, &self.hummock_version_stats);
-                if !node.command_ctx.kind.is_checkpoint() {
+                if !node.command_ctx.barrier_info.kind.is_checkpoint() {
                     assert!(finished_jobs.is_empty());
                     node.notifiers.into_iter().for_each(|notifier| {
                         notifier.notify_collected();
                     });
-                    if let Some(scheduled_barriers) = &mut scheduled_barriers
+                    if let Some((scheduled_barriers, _)) = &mut context
                         && self.create_mview_tracker.has_pending_finished_jobs()
                         && self
                             .command_ctx_queue
                             .values()
-                            .all(|node| !node.command_ctx.kind.is_checkpoint())
+                            .all(|node| !node.command_ctx.barrier_info.kind.is_checkpoint())
                     {
                         scheduled_barriers.force_checkpoint_in_next_barrier();
                     }
                     continue;
                 }
-                let commit_info = collect_commit_epoch_info(
-                    take(&mut node.state.resps),
-                    &node.command_ctx,
-                    self.collect_backfill_pinned_upstream_log_epoch(),
-                );
-                let table_ids_to_finish = node
-                    .state
+                node.state
                     .finished_jobs
                     .drain()
-                    .map(|(table_id, info)| {
+                    .for_each(|(_, (info, resps))| {
+                        node.state.resps.extend(resps);
                         finished_jobs.push(TrackingJob::New(TrackingCommand {
                             info,
                             replace_table_info: None,
                         }));
-                        table_id
-                    })
-                    .collect();
+                    });
+                let commit_info = collect_commit_epoch_info(
+                    take(&mut node.state.resps),
+                    &node.command_ctx,
+                    self.collect_backfill_pinned_upstream_log_epoch(),
+                );
+                self.completing_barrier = Some((
+                    node.command_ctx.barrier_info.prev_epoch(),
+                    node.command_ctx
+                        .command
+                        .as_ref()
+                        .map(|c| c.should_pause_inject_barrier())
+                        .unwrap_or(false),
+                ));
                 task = Some(CompleteBarrierTask {
                     commit_info,
                     finished_jobs,
                     notifiers: node.notifiers,
                     command_context: Some((node.command_ctx, node.enqueue_time)),
-                    table_ids_to_finish,
                     creating_job_epochs: vec![],
                 });
                 break;
             }
         }
-        {
-            {
-                for (table_id, job) in &mut self.creating_streaming_job_controls {
-                    let (upstream_epochs_to_notify, commit_info) = job.start_completing();
-                    for upstream_epoch in upstream_epochs_to_notify {
-                        let wait_progress_timer = self
-                            .command_ctx_queue
-                            .get_mut(&upstream_epoch)
-                            .expect("should exist")
-                            .state
-                            .creating_jobs_to_wait
-                            .remove(table_id)
-                            .expect("should exist");
-                        if let Some(timer) = wait_progress_timer {
-                            timer.observe_duration();
-                        }
-                    }
-                    if let Some((epoch, resps, is_first_time)) = commit_info {
-                        let task = task.get_or_insert_default();
-                        GlobalBarrierManagerContext::collect_creating_job_commit_epoch_info(
-                            &mut task.commit_info,
-                            epoch,
-                            resps,
-                            job.info.table_fragments.all_table_ids().map(TableId::new),
-                            is_first_time,
-                        );
-                        task.creating_job_epochs.push((*table_id, epoch));
-                    }
-                }
+        if !creating_jobs_task.is_empty() {
+            let task = task.get_or_insert_default();
+            for (table_id, epoch, resps, is_first_time) in creating_jobs_task {
+                GlobalBarrierWorkerContextImpl::collect_creating_job_commit_epoch_info(
+                    &mut task.commit_info,
+                    epoch,
+                    resps,
+                    self.creating_streaming_job_controls[&table_id]
+                        .info
+                        .table_fragments
+                        .all_table_ids()
+                        .map(TableId::new),
+                    is_first_time,
+                );
+                task.creating_job_epochs.push((table_id, epoch));
             }
         }
         task
     }
+}
 
+impl CompletingTask {
     pub(super) fn next_completed_barrier<'a>(
         &'a mut self,
-        scheduled_barriers: &mut ScheduledBarriers,
+        scheduled_barriers: &mut PeriodicBarriers,
+        checkpoint_control: &mut CheckpointControl,
+        control_stream_manager: &mut ControlStreamManager,
+        context: &Arc<impl GlobalBarrierWorkerContext>,
+        env: &MetaSrvEnv,
     ) -> impl Future<Output = MetaResult<BarrierCompleteOutput>> + 'a {
         // If there is no completing barrier, try to start completing the earliest barrier if
         // it has been collected.
-        if let CompletingTask::None = &self.completing_task {
-            if let Some(task) = self.next_complete_barrier_task(Some(scheduled_barriers)) {
+        if let CompletingTask::None = self {
+            if let Some(task) = checkpoint_control
+                .next_complete_barrier_task(Some((scheduled_barriers, control_stream_manager)))
+            {
                 {
-                    let command_ctx = task
+                    let creating_job_epochs = task.creating_job_epochs.clone();
+                    let command_prev_epoch = task
                         .command_context
                         .as_ref()
-                        .map(|(command_ctx, _)| command_ctx.clone());
-                    let table_ids_to_finish = task.table_ids_to_finish.clone();
-                    let creating_job_epochs = task.creating_job_epochs.clone();
-                    let join_handle = tokio::spawn(self.context.clone().complete_barrier(task));
-                    self.completing_task = CompletingTask::Completing {
-                        command_ctx,
+                        .map(|(command, _)| command.barrier_info.prev_epoch());
+                    let context = context.clone();
+                    let env = env.clone();
+                    let join_handle =
+                        tokio::spawn(async move { task.complete_barrier(&*context, env).await });
+                    *self = CompletingTask::Completing {
+                        command_prev_epoch,
                         join_handle,
-                        table_ids_to_finish,
                         creating_job_epochs,
                     };
                 }
@@ -1388,11 +1544,11 @@ impl CheckpointControl {
     }
 
     async fn next_completed_barrier_inner(&mut self) -> MetaResult<BarrierCompleteOutput> {
-        let CompletingTask::Completing { join_handle, .. } = &mut self.completing_task else {
+        let CompletingTask::Completing { join_handle, .. } = self else {
             return pending().await;
         };
 
-        let (table_ids_to_finish, creating_job_epochs) = {
+        {
             {
                 let join_result: MetaResult<_> = try {
                     join_handle
@@ -1406,64 +1562,46 @@ impl CheckpointControl {
                 } else {
                     CompletingTask::None
                 };
-                let completed_command =
-                    replace(&mut self.completing_task, next_completing_command_status);
-                self.hummock_version_stats = join_result?;
+                let completed_command = replace(self, next_completing_command_status);
+                let hummock_version_stats = join_result?;
 
                 must_match!(completed_command, CompletingTask::Completing {
-                    table_ids_to_finish,
                     creating_job_epochs,
+                    command_prev_epoch,
                     ..
-                } => (table_ids_to_finish, creating_job_epochs))
+                } => {
+                    Ok(BarrierCompleteOutput {
+                        command_prev_epoch,
+                        creating_job_epochs,
+                        hummock_version_stats,
+                    })
+                })
             }
-        };
+        }
+    }
+}
 
+impl CheckpointControl {
+    fn ack_completed(&mut self, output: BarrierCompleteOutput) {
         {
-            for (table_id, epoch) in creating_job_epochs {
-                if let Some((upstream_epoch, is_finished)) = self
-                    .creating_streaming_job_controls
+            self.hummock_version_stats = output.hummock_version_stats;
+            assert_eq!(
+                self.completing_barrier
+                    .take()
+                    .map(|(prev_epoch, _)| prev_epoch),
+                output.command_prev_epoch
+            );
+            for (table_id, epoch) in output.creating_job_epochs {
+                self.creating_streaming_job_controls
                     .get_mut(&table_id)
                     .expect("should exist")
                     .ack_completed(epoch)
-                {
-                    let wait_progress_timer = self
-                        .command_ctx_queue
-                        .get_mut(&upstream_epoch)
-                        .expect("should exist")
-                        .state
-                        .creating_jobs_to_wait
-                        .remove(&table_id)
-                        .expect("should exist");
-                    if let Some(timer) = wait_progress_timer {
-                        timer.observe_duration();
-                    }
-                    if is_finished {
-                        debug!(epoch, ?table_id, "finish creating job");
-                        let creating_streaming_job = self
-                            .creating_streaming_job_controls
-                            .remove(&table_id)
-                            .expect("should exist");
-                        assert!(creating_streaming_job.is_finished());
-                        assert!(self
-                            .command_ctx_queue
-                            .get_mut(&upstream_epoch)
-                            .expect("should exist")
-                            .state
-                            .finished_jobs
-                            .insert(table_id, creating_streaming_job.info)
-                            .is_none());
-                    }
-                }
             }
-
-            Ok(BarrierCompleteOutput {
-                table_ids_to_finish,
-            })
         }
     }
 }
 
-impl GlobalBarrierManagerContext {
+impl GlobalBarrierManager {
     /// Check the status of barrier manager, return error if it is not `Running`.
     pub fn check_status_running(&self) -> MetaResult<()> {
         let status = self.status.load();
@@ -1485,12 +1623,9 @@ impl GlobalBarrierManagerContext {
     pub fn get_recovery_status(&self) -> PbRecoveryStatus {
         (&**self.status.load()).into()
     }
+}
 
-    /// Set barrier manager status.
-    fn set_status(&self, new_status: BarrierManagerStatus) {
-        self.status.store(Arc::new(new_status));
-    }
-
+impl GlobalBarrierWorkerContextImpl {
     /// Resolve actor information from cluster, fragment manager and `ChangedTableId`.
     /// We use `changed_table_id` to modify the actors to be sent or collected. Because these actor
     /// will create or drop before this barrier flow through them.
@@ -1501,15 +1636,20 @@ impl GlobalBarrierManagerContext {
             .load_all_actors()
             .await?;
 
-        Ok(InflightGraphInfo::new(
-            all_actor_infos
-                .fragment_infos
-                .into_iter()
-                .map(|(id, info)| (id as _, info))
-                .collect(),
-        ))
+        Ok(InflightGraphInfo::new(all_actor_infos))
     }
+}
+
+pub struct GlobalBarrierManager {
+    status: Arc<ArcSwap<BarrierManagerStatus>>,
+    hummock_manager: HummockManagerRef,
+    request_tx: mpsc::UnboundedSender<BarrierManagerRequest>,
+    metadata_manager: MetadataManager,
+}
+
+pub type BarrierManagerRef = Arc<GlobalBarrierManager>;
 
+impl GlobalBarrierManager {
     /// Serving `SHOW JOBS / SELECT * FROM rw_ddl_progress`
     pub async fn get_ddl_progress(&self) -> MetaResult<Vec<DdlProgress>> {
         let mut ddl_progress = {
@@ -1539,9 +1679,11 @@ impl GlobalBarrierManagerContext {
 
         Ok(ddl_progress.into_values().collect())
     }
-}
 
-pub type BarrierManagerRef = GlobalBarrierManagerContext;
+    pub async fn get_hummock_version_id(&self) -> HummockVersionId {
+        self.hummock_manager.get_version_id().await
+    }
+}
 
 #[expect(clippy::type_complexity)]
 fn collect_resp_info(
@@ -1600,19 +1742,21 @@ fn collect_commit_epoch_info(
     let (sst_to_context, synced_ssts, new_table_watermarks, old_value_ssts) =
         collect_resp_info(resps);
 
-    let new_table_fragment_infos = if let Command::CreateStreamingJob { info, job_type } =
+    let new_table_fragment_infos = if let Some(Command::CreateStreamingJob { info, job_type }) =
         &command_ctx.command
         && !matches!(job_type, CreateStreamingJobType::SnapshotBackfill(_))
     {
         let table_fragments = &info.table_fragments;
-        vec![NewTableFragmentInfo::Normal {
-            mv_table_id: table_fragments.mv_table_id().map(TableId::new),
-            internal_table_ids: table_fragments
-                .internal_table_ids()
-                .into_iter()
-                .map(TableId::new)
-                .collect(),
-        }]
+        let mut table_ids: HashSet<_> = table_fragments
+            .internal_table_ids()
+            .into_iter()
+            .map(TableId::new)
+            .collect();
+        if let Some(mv_table_id) = table_fragments.mv_table_id() {
+            table_ids.insert(TableId::new(mv_table_id));
+        }
+
+        vec![NewTableFragmentInfo { table_ids }]
     } else {
         vec![]
     };
@@ -1650,11 +1794,11 @@ fn collect_commit_epoch_info(
     let table_new_change_log = build_table_change_log_delta(
         old_value_ssts.into_iter(),
         synced_ssts.iter().map(|sst| &sst.sst_info),
-        must_match!(&command_ctx.kind, BarrierKind::Checkpoint(epochs) => epochs),
+        must_match!(&command_ctx.barrier_info.kind, BarrierKind::Checkpoint(epochs) => epochs),
         mv_log_store_truncate_epoch.into_iter(),
     );
 
-    let epoch = command_ctx.prev_epoch.value().0;
+    let epoch = command_ctx.barrier_info.prev_epoch();
     let tables_to_commit = command_ctx
         .table_ids_to_commit
         .iter()
diff --git a/src/meta/src/barrier/progress.rs b/src/meta/src/barrier/progress.rs
index e28d41ef09b5e..9cb163d3448b5 100644
--- a/src/meta/src/barrier/progress.rs
+++ b/src/meta/src/barrier/progress.rs
@@ -28,7 +28,7 @@ use crate::barrier::{
     Command, CreateStreamingJobCommandInfo, CreateStreamingJobType, EpochNode, ReplaceTablePlan,
 };
 use crate::manager::{DdlType, MetadataManager};
-use crate::model::{ActorId, TableFragments};
+use crate::model::{ActorId, BackfillUpstreamType, TableFragments};
 use crate::MetaResult;
 
 type ConsumedRows = u64;
@@ -36,28 +36,29 @@ type ConsumedRows = u64;
 #[derive(Clone, Copy, Debug)]
 enum BackfillState {
     Init,
-    ConsumingUpstream(#[allow(dead_code)] Epoch, ConsumedRows),
+    ConsumingUpstream(#[expect(dead_code)] Epoch, ConsumedRows),
     Done(ConsumedRows),
 }
 
 /// Progress of all actors containing backfill executors while creating mview.
 #[derive(Debug)]
 pub(super) struct Progress {
+    // `states` and `done_count` decides whether the progress is done. See `is_done`.
     states: HashMap<ActorId, BackfillState>,
-
     done_count: usize,
 
+    /// Tells whether the backfill is from source or mv.
+    backfill_upstream_types: HashMap<ActorId, BackfillUpstreamType>,
+
+    // The following row counts are used to calculate the progress. See `calculate_progress`.
     /// Upstream mv count.
     /// Keep track of how many times each upstream MV
     /// appears in this stream job.
     upstream_mv_count: HashMap<TableId, usize>,
-
-    /// Total key count in the upstream materialized view
-    /// TODO: implement this for source backfill
-    upstream_total_key_count: u64,
-
-    /// Consumed rows
-    consumed_rows: u64,
+    /// Total key count of all the upstream materialized views
+    upstream_mvs_total_key_count: u64,
+    mv_backfill_consumed_rows: u64,
+    source_backfill_consumed_rows: u64,
 
     /// DDL definition
     definition: String,
@@ -66,47 +67,55 @@ pub(super) struct Progress {
 impl Progress {
     /// Create a [`Progress`] for some creating mview, with all `actors` containing the backfill executors.
     fn new(
-        actors: impl IntoIterator<Item = ActorId>,
+        actors: impl IntoIterator<Item = (ActorId, BackfillUpstreamType)>,
         upstream_mv_count: HashMap<TableId, usize>,
         upstream_total_key_count: u64,
         definition: String,
     ) -> Self {
-        let states = actors
-            .into_iter()
-            .map(|a| (a, BackfillState::Init))
-            .collect::<HashMap<_, _>>();
+        let mut states = HashMap::new();
+        let mut backfill_upstream_types = HashMap::new();
+        for (actor, backfill_upstream_type) in actors {
+            states.insert(actor, BackfillState::Init);
+            backfill_upstream_types.insert(actor, backfill_upstream_type);
+        }
         assert!(!states.is_empty());
 
         Self {
             states,
+            backfill_upstream_types,
             done_count: 0,
             upstream_mv_count,
-            upstream_total_key_count,
-            consumed_rows: 0,
+            upstream_mvs_total_key_count: upstream_total_key_count,
+            mv_backfill_consumed_rows: 0,
+            source_backfill_consumed_rows: 0,
             definition,
         }
     }
 
     /// Update the progress of `actor`.
     fn update(&mut self, actor: ActorId, new_state: BackfillState, upstream_total_key_count: u64) {
-        self.upstream_total_key_count = upstream_total_key_count;
+        self.upstream_mvs_total_key_count = upstream_total_key_count;
         let total_actors = self.states.len();
+        let backfill_upstream_type = self.backfill_upstream_types.get(&actor).unwrap();
         tracing::debug!(?actor, states = ?self.states, "update progress for actor");
+
+        let mut old = 0;
+        let mut new = 0;
         match self.states.remove(&actor).unwrap() {
             BackfillState::Init => {}
             BackfillState::ConsumingUpstream(_, old_consumed_rows) => {
-                self.consumed_rows -= old_consumed_rows;
+                old = old_consumed_rows;
             }
             BackfillState::Done(_) => panic!("should not report done multiple times"),
         };
         match &new_state {
             BackfillState::Init => {}
             BackfillState::ConsumingUpstream(_, new_consumed_rows) => {
-                self.consumed_rows += new_consumed_rows;
+                new = *new_consumed_rows;
             }
             BackfillState::Done(new_consumed_rows) => {
                 tracing::debug!("actor {} done", actor);
-                self.consumed_rows += new_consumed_rows;
+                new = *new_consumed_rows;
                 self.done_count += 1;
                 tracing::debug!(
                     "{} actors out of {} complete",
@@ -115,8 +124,19 @@ impl Progress {
                 );
             }
         };
+        debug_assert!(new >= old, "backfill progress should not go backward");
+        match backfill_upstream_type {
+            BackfillUpstreamType::MView => {
+                self.mv_backfill_consumed_rows += new - old;
+            }
+            BackfillUpstreamType::Source => {
+                self.source_backfill_consumed_rows += new - old;
+            }
+            BackfillUpstreamType::Values => {
+                // do not consider progress for values
+            }
+        }
         self.states.insert(actor, new_state);
-        self.calculate_progress();
     }
 
     /// Returns whether all backfill executors are done.
@@ -137,19 +157,52 @@ impl Progress {
     }
 
     /// `progress` = `consumed_rows` / `upstream_total_key_count`
-    fn calculate_progress(&self) -> f64 {
+    fn calculate_progress(&self) -> String {
         if self.is_done() || self.states.is_empty() {
-            return 1.0;
+            return "100%".to_string();
         }
-        let mut upstream_total_key_count = self.upstream_total_key_count as f64;
-        if upstream_total_key_count == 0.0 {
-            upstream_total_key_count = 1.0
+        let mut mv_count = 0;
+        let mut source_count = 0;
+        for backfill_upstream_type in self.backfill_upstream_types.values() {
+            match backfill_upstream_type {
+                BackfillUpstreamType::MView => mv_count += 1,
+                BackfillUpstreamType::Source => source_count += 1,
+                BackfillUpstreamType::Values => (),
+            }
         }
-        let mut progress = self.consumed_rows as f64 / upstream_total_key_count;
-        if progress >= 1.0 {
-            progress = 0.99;
+
+        let mv_progress = (mv_count > 0).then_some({
+            if self.upstream_mvs_total_key_count == 0 {
+                "99.99%".to_string()
+            } else {
+                let mut progress = self.mv_backfill_consumed_rows as f64
+                    / (self.upstream_mvs_total_key_count as f64);
+                if progress > 1.0 {
+                    progress = 0.9999;
+                }
+                format!(
+                    "{:.2}% ({}/{})",
+                    progress * 100.0,
+                    self.mv_backfill_consumed_rows,
+                    self.upstream_mvs_total_key_count
+                )
+            }
+        });
+        let source_progress = (source_count > 0).then_some(format!(
+            "{} rows consumed",
+            self.source_backfill_consumed_rows
+        ));
+        match (mv_progress, source_progress) {
+            (Some(mv_progress), Some(source_progress)) => {
+                format!(
+                    "MView Backfill: {}, Source Backfill: {}",
+                    mv_progress, source_progress
+                )
+            }
+            (Some(mv_progress), None) => mv_progress,
+            (None, Some(source_progress)) => source_progress,
+            (None, None) => "Unknown".to_string(),
         }
-        progress
     }
 }
 
@@ -180,8 +233,7 @@ impl TrackingJob {
                 Ok(())
             }
             TrackingJob::Recovered(recovered) => {
-                recovered
-                    .metadata_manager
+                metadata_manager
                     .catalog_controller
                     .finish_streaming_job(recovered.id, None)
                     .await?;
@@ -215,7 +267,6 @@ impl std::fmt::Debug for TrackingJob {
 
 pub struct RecoveredTrackingJob {
     pub id: ObjectId,
-    pub metadata_manager: MetadataManager,
 }
 
 /// The command tracking by the [`CreateMviewProgressTracker`].
@@ -231,11 +282,9 @@ pub(super) struct TrackingCommand {
 /// 4. With `actor_map` we can use an actor's `ActorId` to find the ID of the `StreamJob`.
 #[derive(Default, Debug)]
 pub(super) struct CreateMviewProgressTracker {
-    // TODO: add a specialized progress for source
     /// Progress of the create-mview DDL indicated by the `TableId`.
     progress_map: HashMap<TableId, (Progress, TrackingJob)>,
 
-    /// Find the epoch of the create-mview DDL by the actor containing the MV/source backfill executors.
     actor_map: HashMap<ActorId, TableId>,
 
     /// Stash of finished jobs. They will be finally finished on checkpoint.
@@ -252,28 +301,29 @@ impl CreateMviewProgressTracker {
     /// 2. `Backfill` position.
     pub fn recover(
         mview_map: HashMap<TableId, (String, TableFragments)>,
-        version_stats: HummockVersionStats,
-        metadata_manager: MetadataManager,
+        version_stats: &HummockVersionStats,
     ) -> Self {
         let mut actor_map = HashMap::new();
         let mut progress_map = HashMap::new();
         for (creating_table_id, (definition, table_fragments)) in mview_map {
             let mut states = HashMap::new();
-            let actors = table_fragments.backfill_actor_ids();
-            for actor in actors {
+            let mut backfill_upstream_types = HashMap::new();
+            let actors = table_fragments.tracking_progress_actor_ids();
+            for (actor, backfill_upstream_type) in actors {
                 actor_map.insert(actor, creating_table_id);
                 states.insert(actor, BackfillState::ConsumingUpstream(Epoch(0), 0));
+                backfill_upstream_types.insert(actor, backfill_upstream_type);
             }
 
             let progress = Self::recover_progress(
                 states,
+                backfill_upstream_types,
                 table_fragments.dependent_table_ids(),
                 definition,
-                &version_stats,
+                version_stats,
             );
             let tracking_job = TrackingJob::Recovered(RecoveredTrackingJob {
                 id: creating_table_id.table_id as i32,
-                metadata_manager: metadata_manager.clone(),
             });
             progress_map.insert(creating_table_id, (progress, tracking_job));
         }
@@ -284,28 +334,28 @@ impl CreateMviewProgressTracker {
         }
     }
 
+    /// ## How recovery works
+    ///
+    /// The progress (number of rows consumed) is persisted in state tables.
+    /// During recovery, the backfill executor will restore the number of rows consumed,
+    /// and then it will just report progress like newly created executors.
     fn recover_progress(
         states: HashMap<ActorId, BackfillState>,
+        backfill_upstream_types: HashMap<ActorId, BackfillUpstreamType>,
         upstream_mv_count: HashMap<TableId, usize>,
         definition: String,
         version_stats: &HummockVersionStats,
     ) -> Progress {
-        let upstream_total_key_count = upstream_mv_count
-            .iter()
-            .map(|(upstream_mv, count)| {
-                *count as u64
-                    * version_stats
-                        .table_stats
-                        .get(&upstream_mv.table_id)
-                        .map_or(0, |stat| stat.total_key_count as u64)
-            })
-            .sum();
+        let upstream_mvs_total_key_count =
+            calculate_total_key_count(&upstream_mv_count, version_stats);
         Progress {
             states,
+            backfill_upstream_types,
             done_count: 0, // Fill only after first barrier pass
             upstream_mv_count,
-            upstream_total_key_count,
-            consumed_rows: 0, // Fill only after first barrier pass
+            upstream_mvs_total_key_count,
+            mv_backfill_consumed_rows: 0, // Fill only after first barrier pass
+            source_backfill_consumed_rows: 0, // Fill only after first barrier pass
             definition,
         }
     }
@@ -318,7 +368,7 @@ impl CreateMviewProgressTracker {
                 let ddl_progress = DdlProgress {
                     id: table_id as u64,
                     statement: x.definition.clone(),
-                    progress: format!("{:.2}%", x.calculate_progress() * 100.0),
+                    progress: x.calculate_progress(),
                 };
                 (table_id, ddl_progress)
             })
@@ -334,7 +384,7 @@ impl CreateMviewProgressTracker {
     ) -> Vec<TrackingJob> {
         let command_ctx = &epoch_node.command_ctx;
         let new_tracking_job_info =
-            if let Command::CreateStreamingJob { info, job_type } = &command_ctx.command {
+            if let Some(Command::CreateStreamingJob { info, job_type }) = &command_ctx.command {
                 match job_type {
                     CreateStreamingJobType::Normal => Some((info, None)),
                     CreateStreamingJobType::SinkIntoTable(replace_table) => {
@@ -358,12 +408,16 @@ impl CreateMviewProgressTracker {
                 .flat_map(|resp| resp.create_mview_progress.iter()),
             version_stats,
         );
-        if let Some(table_id) = command_ctx.command.table_to_cancel() {
+        if let Some(table_id) = command_ctx
+            .command
+            .as_ref()
+            .and_then(Command::table_to_cancel)
+        {
             // the cancelled command is possibly stashed in `finished_commands` and waiting
             // for checkpoint, we should also clear it.
             self.cancel_command(table_id);
         }
-        if command_ctx.kind.is_checkpoint() {
+        if command_ctx.barrier_info.kind.is_checkpoint() {
             self.take_finished_jobs()
         } else {
             vec![]
@@ -377,7 +431,7 @@ impl CreateMviewProgressTracker {
 
     /// Finish stashed jobs on checkpoint.
     pub(super) fn take_finished_jobs(&mut self) -> Vec<TrackingJob> {
-        tracing::trace!(finished_jobs=?self.pending_finished_jobs, progress_map=?self.progress_map, "finishing jobs");
+        tracing::trace!(finished_jobs=?self.pending_finished_jobs, progress_map=?self.progress_map, "take_finished_jobs");
         take(&mut self.pending_finished_jobs)
     }
 
@@ -449,16 +503,8 @@ impl CreateMviewProgressTracker {
                 upstream_mv_count.insert(*table_id, dispatch_count / actors.len());
             }
 
-            let upstream_total_key_count: u64 = upstream_mv_count
-                .iter()
-                .map(|(upstream_mv, count)| {
-                    *count as u64
-                        * version_stats
-                            .table_stats
-                            .get(&upstream_mv.table_id)
-                            .map_or(0, |stat| stat.total_key_count as u64)
-                })
-                .sum();
+            let upstream_total_key_count: u64 =
+                calculate_total_key_count(&upstream_mv_count, version_stats);
             (
                 upstream_mv_count,
                 upstream_total_key_count,
@@ -467,8 +513,8 @@ impl CreateMviewProgressTracker {
             )
         };
 
-        for &actor in &actors {
-            self.actor_map.insert(actor, creating_mv_id);
+        for (actor, _backfill_upstream_type) in &actors {
+            self.actor_map.insert(*actor, creating_mv_id);
         }
 
         let progress = Progress::new(
@@ -537,18 +583,8 @@ impl CreateMviewProgressTracker {
             Entry::Occupied(mut o) => {
                 let progress = &mut o.get_mut().0;
 
-                let upstream_total_key_count: u64 = progress
-                    .upstream_mv_count
-                    .iter()
-                    .map(|(upstream_mv, count)| {
-                        assert_ne!(*count, 0);
-                        *count as u64
-                            * version_stats
-                                .table_stats
-                                .get(&upstream_mv.table_id)
-                                .map_or(0, |stat| stat.total_key_count as u64)
-                    })
-                    .sum();
+                let upstream_total_key_count: u64 =
+                    calculate_total_key_count(&progress.upstream_mv_count, version_stats);
 
                 tracing::debug!(?table_id, "updating progress for table");
                 progress.update(actor, new_state, upstream_total_key_count);
@@ -577,3 +613,20 @@ impl CreateMviewProgressTracker {
         }
     }
 }
+
+fn calculate_total_key_count(
+    table_count: &HashMap<TableId, usize>,
+    version_stats: &HummockVersionStats,
+) -> u64 {
+    table_count
+        .iter()
+        .map(|(table_id, count)| {
+            assert_ne!(*count, 0);
+            *count as u64
+                * version_stats
+                    .table_stats
+                    .get(&table_id.table_id)
+                    .map_or(0, |stat| stat.total_key_count as u64)
+        })
+        .sum()
+}
diff --git a/src/meta/src/barrier/recovery.rs b/src/meta/src/barrier/recovery.rs
index 498950a98014f..99ad09a4a1694 100644
--- a/src/meta/src/barrier/recovery.rs
+++ b/src/meta/src/barrier/recovery.rs
@@ -17,11 +17,14 @@ use std::time::Duration;
 
 use anyhow::{anyhow, Context};
 use itertools::Itertools;
+use risingwave_common::bail;
 use risingwave_common::catalog::TableId;
 use risingwave_common::config::DefaultParallelism;
 use risingwave_common::hash::WorkerSlotId;
 use risingwave_common::util::epoch::Epoch;
+use risingwave_connector::source::SplitImpl;
 use risingwave_meta_model::{StreamingParallelism, WorkerId};
+use risingwave_pb::hummock::HummockVersionStats;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::{PausedReason, Recovery};
 use risingwave_pb::stream_plan::barrier_mutation::Mutation;
@@ -29,23 +32,25 @@ use risingwave_pb::stream_plan::{AddMutation, StreamActor};
 use thiserror_ext::AsReport;
 use tokio::time::Instant;
 use tokio_retry::strategy::{jitter, ExponentialBackoff};
-use tracing::{debug, error, info, warn, Instrument};
+use tracing::{debug, info, warn, Instrument};
 
-use super::{CheckpointControl, TracedEpoch};
-use crate::barrier::info::{InflightGraphInfo, InflightSubscriptionInfo};
+use super::{
+    CheckpointControl, GlobalBarrierWorker, GlobalBarrierWorkerContext, RecoveryReason, TracedEpoch,
+};
+use crate::barrier::info::{BarrierInfo, InflightGraphInfo, InflightSubscriptionInfo};
 use crate::barrier::progress::CreateMviewProgressTracker;
 use crate::barrier::rpc::ControlStreamManager;
 use crate::barrier::schedule::ScheduledBarriers;
-use crate::barrier::state::BarrierManagerState;
-use crate::barrier::{BarrierKind, GlobalBarrierManager, GlobalBarrierManagerContext};
+use crate::barrier::state::BarrierWorkerState;
+use crate::barrier::{BarrierKind, GlobalBarrierWorkerContextImpl};
+use crate::controller::fragment::InflightFragmentInfo;
 use crate::manager::ActiveStreamingWorkerNodes;
-use crate::model::{TableFragments, TableParallelism};
+use crate::model::{ActorId, TableFragments, TableParallelism};
+use crate::rpc::metrics::GLOBAL_META_METRICS;
 use crate::stream::{build_actor_connector_splits, RescheduleOptions, TableResizePolicy};
 use crate::{model, MetaError, MetaResult};
 
-impl GlobalBarrierManager {
-    // Migration timeout.
-    const RECOVERY_FORCE_MIGRATION_TIMEOUT: Duration = Duration::from_secs(300);
+impl<C> GlobalBarrierWorker<C> {
     // Retry base interval in milliseconds.
     const RECOVERY_RETRY_BASE_INTERVAL: u64 = 20;
     // Retry max interval.
@@ -60,7 +65,7 @@ impl GlobalBarrierManager {
     }
 }
 
-impl GlobalBarrierManagerContext {
+impl GlobalBarrierWorkerContextImpl {
     /// Clean catalogs for creating streaming jobs that are in foreground mode or table fragments not persisted.
     async fn clean_dirty_streaming_jobs(&self) -> MetaResult<()> {
         self.metadata_manager
@@ -87,7 +92,10 @@ impl GlobalBarrierManagerContext {
         Ok(())
     }
 
-    async fn recover_background_mv_progress(&self) -> MetaResult<CreateMviewProgressTracker> {
+    // FIXME: didn't consider Values here
+    async fn recover_background_mv_progress(
+        &self,
+    ) -> MetaResult<HashMap<TableId, (String, TableFragments)>> {
         let mgr = &self.metadata_manager;
         let mviews = mgr
             .catalog_controller
@@ -105,25 +113,22 @@ impl GlobalBarrierManagerContext {
             mview_map.insert(table_id, (mview.definition.clone(), table_fragments));
         }
 
-        let version_stats = self.hummock_manager.get_version_stats().await;
         // If failed, enter recovery mode.
 
-        Ok(CreateMviewProgressTracker::recover(
-            mview_map,
-            version_stats,
-            mgr.clone(),
-        ))
+        Ok(mview_map)
     }
+}
 
+impl ScheduledBarriers {
     /// Pre buffered drop and cancel command, return true if any.
-    fn pre_apply_drop_cancel(&self, scheduled_barriers: &ScheduledBarriers) -> bool {
-        let (dropped_actors, cancelled) = scheduled_barriers.pre_apply_drop_cancel_scheduled();
+    fn pre_apply_drop_cancel(&self) -> bool {
+        let (dropped_actors, cancelled) = self.pre_apply_drop_cancel_scheduled();
 
         !dropped_actors.is_empty() || !cancelled.is_empty()
     }
 }
 
-impl GlobalBarrierManager {
+impl<C: GlobalBarrierWorkerContext> GlobalBarrierWorker<C> {
     /// Recovery the whole cluster from the latest epoch.
     ///
     /// If `paused_reason` is `Some`, all data sources (including connectors and DMLs) will be
@@ -131,56 +136,57 @@ impl GlobalBarrierManager {
     /// the cluster or `risectl` command. Used for debugging purpose.
     ///
     /// Returns the new state of the barrier manager after recovery.
-    pub async fn recovery(&mut self, paused_reason: Option<PausedReason>, err: Option<MetaError>) {
-        // Mark blocked and abort buffered schedules, they might be dirty already.
-        self.scheduled_barriers
-            .abort_and_mark_blocked("cluster is under recovering");
+    pub async fn recovery(
+        &mut self,
+        paused_reason: Option<PausedReason>,
+        err: Option<MetaError>,
+        recovery_reason: RecoveryReason,
+    ) {
+        self.context.abort_and_mark_blocked(recovery_reason);
         // Clear all control streams to release resources (connections to compute nodes) first.
         self.control_stream_manager.clear();
 
-        tracing::info!("recovery start!");
-        let retry_strategy = Self::get_retry_strategy();
-
-        // We take retry into consideration because this is the latency user sees for a cluster to
-        // get recovered.
-        let recovery_timer = self.context.metrics.recovery_latency.start_timer();
-
-        let new_state = tokio_retry::Retry::spawn(retry_strategy, || {
-            async {
-                let recovery_result: MetaResult<_> = try {
-                    if let Some(err) = &err {
-                        self.context
-                            .metadata_manager
-                            .notify_finish_failed(err)
-                            .await;
-                    }
+        self.recovery_inner(paused_reason, err).await;
+        self.context.mark_ready();
+    }
+}
 
-                    self.context
-                        .clean_dirty_streaming_jobs()
+impl GlobalBarrierWorkerContextImpl {
+    pub(super) async fn reload_runtime_info_impl(
+        &self,
+    ) -> MetaResult<(
+        ActiveStreamingWorkerNodes,
+        InflightGraphInfo,
+        InflightSubscriptionInfo,
+        Option<TracedEpoch>,
+        HashMap<WorkerId, Vec<StreamActor>>,
+        HashMap<ActorId, Vec<SplitImpl>>,
+        HashMap<TableId, (String, TableFragments)>,
+        HummockVersionStats,
+    )> {
+        {
+            {
+                {
+                    self.clean_dirty_streaming_jobs()
                         .await
                         .context("clean dirty streaming jobs")?;
 
                     // Mview progress needs to be recovered.
                     tracing::info!("recovering mview progress");
-                    let tracker = self
-                        .context
+                    let background_mview_progress = self
                         .recover_background_mv_progress()
                         .await
                         .context("recover mview progress should not fail")?;
                     tracing::info!("recovered mview progress");
 
                     // This is a quick path to accelerate the process of dropping and canceling streaming jobs.
-                    let _ = self
-                        .context
-                        .pre_apply_drop_cancel(&self.scheduled_barriers);
+                    let _ = self.scheduled_barriers.pre_apply_drop_cancel();
 
-                    let mut active_streaming_nodes = ActiveStreamingWorkerNodes::new_snapshot(
-                        self.context.metadata_manager.clone(),
-                    )
-                    .await?;
+                    let mut active_streaming_nodes =
+                        ActiveStreamingWorkerNodes::new_snapshot(self.metadata_manager.clone())
+                            .await?;
 
                     let background_streaming_jobs = self
-                        .context
                         .metadata_manager
                         .list_background_creating_jobs()
                         .await?;
@@ -192,44 +198,37 @@ impl GlobalBarrierManager {
                     let mut info = if !self.env.opts.disable_automatic_parallelism_control
                         && background_streaming_jobs.is_empty()
                     {
-                        self.context
-                            .scale_actors(&active_streaming_nodes)
+                        self.scale_actors(&active_streaming_nodes)
                             .await
                             .inspect_err(|err| {
                                 warn!(error = %err.as_report(), "scale actors failed");
                             })?;
 
-                        self.context.resolve_graph_info().await.inspect_err(|err| {
+                        self.resolve_graph_info().await.inspect_err(|err| {
                             warn!(error = %err.as_report(), "resolve actor info failed");
                         })?
                     } else {
                         // Migrate actors in expired CN to newly joined one.
-                        self.context
-                            .migrate_actors(&mut active_streaming_nodes)
+                        self.migrate_actors(&mut active_streaming_nodes)
                             .await
                             .inspect_err(|err| {
                                 warn!(error = %err.as_report(), "migrate actors failed");
                             })?
                     };
 
-                    if self
-                        .context
-                        .pre_apply_drop_cancel(&self.scheduled_barriers)
-                    {
-                        info = self.context.resolve_graph_info().await.inspect_err(|err| {
+                    if self.scheduled_barriers.pre_apply_drop_cancel() {
+                        info = self.resolve_graph_info().await.inspect_err(|err| {
                             warn!(error = %err.as_report(), "resolve actor info failed");
                         })?
                     }
 
                     let info = info;
 
-                    self.context
-                        .purge_state_table_from_hummock(&info.existing_table_ids().collect())
+                    self.purge_state_table_from_hummock(&info.existing_table_ids().collect())
                         .await
                         .context("purge state table from hummock")?;
 
-                    let (prev_epoch, version_id) = self
-                        .context
+                    let prev_epoch = self
                         .hummock_manager
                         .on_current_version(|version| {
                             let state_table_info = version.state_table_info.info();
@@ -255,44 +254,23 @@ impl GlobalBarrierManager {
                                     );
                                 }
                             }
-                            (
-                                committed_epoch.map(|committed_epoch| {
-                                    TracedEpoch::new(Epoch::from(committed_epoch))
-                                }),
-                                version.id,
-                            )
+                            committed_epoch.map(|committed_epoch| {
+                                TracedEpoch::new(Epoch::from(committed_epoch))
+                            })
                         })
                         .await;
 
-                    let mut control_stream_manager =
-                        ControlStreamManager::new(self.context.clone());
-
                     let subscription_info = InflightSubscriptionInfo {
                         mv_depended_subscriptions: self
-                            .context
                             .metadata_manager
                             .get_mv_depended_subscriptions()
                             .await?,
                     };
 
-                    let reset_start_time = Instant::now();
-                    control_stream_manager
-                        .reset(
-                            version_id,
-                            &subscription_info,
-                            active_streaming_nodes.current(),
-                        )
-                        .await
-                        .inspect_err(|err| {
-                            warn!(error = %err.as_report(), "reset compute nodes failed");
-                        })?;
-                    info!(elapsed=?reset_start_time.elapsed(), "control stream reset");
-
-                    self.context.sink_manager.reset().await;
+                    self.sink_manager.reset().await;
 
                     // update and build all actors.
                     let node_actors = self
-                        .context
                         .load_all_actors(&info, &active_streaming_nodes)
                         .await
                         .inspect_err(|err| {
@@ -300,8 +278,67 @@ impl GlobalBarrierManager {
                         })?;
 
                     // get split assignments for all actors
-                    let source_split_assignments =
-                        self.context.source_manager.list_assignments().await;
+                    let source_split_assignments = self.source_manager.list_assignments().await;
+                    Ok((
+                        active_streaming_nodes,
+                        info,
+                        subscription_info,
+                        prev_epoch,
+                        node_actors,
+                        source_split_assignments,
+                        background_mview_progress,
+                        self.hummock_manager.get_version_stats().await,
+                    ))
+                }
+            }
+        }
+    }
+}
+
+impl<C: GlobalBarrierWorkerContext> GlobalBarrierWorker<C> {
+    async fn recovery_inner(
+        &mut self,
+        paused_reason: Option<PausedReason>,
+        err: Option<MetaError>,
+    ) {
+        tracing::info!("recovery start!");
+        let retry_strategy = Self::get_retry_strategy();
+
+        // We take retry into consideration because this is the latency user sees for a cluster to
+        // get recovered.
+        let recovery_timer = GLOBAL_META_METRICS.recovery_latency.start_timer();
+
+        let new_state = tokio_retry::Retry::spawn(retry_strategy, || async {
+            if let Some(err) = &err {
+                self.context.notify_creating_job_failed(err).await;
+            };
+            let (
+                active_streaming_nodes,
+                info,
+                subscription_info,
+                prev_epoch,
+                node_actors,
+                source_split_assignments,
+                background_mview_progress,
+                version_stats,
+            ) = self.context.reload_runtime_info().await?;
+
+            let mut control_stream_manager = ControlStreamManager::new(self.env.clone());
+            let reset_start_time = Instant::now();
+            control_stream_manager
+                .reset(
+                    &subscription_info,
+                    active_streaming_nodes.current(),
+                    &*self.context,
+                )
+                .await
+                .inspect_err(|err| {
+                    warn!(error = %err.as_report(), "reset compute nodes failed");
+                })?;
+            info!(elapsed=?reset_start_time.elapsed(), "control stream reset");
+
+            let recovery_result: MetaResult<_> = try {
+                {
                     let mutation = Mutation::Add(AddMutation {
                         // Actors built during recovery is not treated as newly added actors.
                         actor_dispatchers: Default::default(),
@@ -311,82 +348,93 @@ impl GlobalBarrierManager {
                         subscriptions_to_add: Default::default(),
                     });
 
-                    let new_epoch = if let Some(prev_epoch) = &prev_epoch {
+                    let new_epoch = if let Some(prev_epoch) = prev_epoch {
                         // Use a different `curr_epoch` for each recovery attempt.
-                        let new_epoch = prev_epoch.next();
+                        let curr_epoch = prev_epoch.next();
+                        let barrier_info = BarrierInfo {
+                            prev_epoch,
+                            curr_epoch,
+                            kind: BarrierKind::Initial,
+                        };
 
                         let mut node_to_collect = control_stream_manager.inject_barrier(
                             None,
                             Some(mutation),
-                            (&new_epoch, prev_epoch),
-                            &BarrierKind::Initial,
-                            &info,
-                            Some(&info),
+                            &barrier_info,
+                            info.fragment_infos(),
+                            info.fragment_infos(),
                             Some(node_actors),
                             vec![],
                             vec![],
                         )?;
                         debug!(?node_to_collect, "inject initial barrier");
                         while !node_to_collect.is_empty() {
-                            let (worker_id, result) = control_stream_manager
-                                .next_complete_barrier_response()
-                                .await;
+                            let (worker_id, result) =
+                                control_stream_manager.next_collect_barrier_response().await;
                             let resp = result?;
-                            assert_eq!(resp.epoch, prev_epoch.value().0);
+                            assert_eq!(resp.epoch, barrier_info.prev_epoch());
                             assert!(node_to_collect.remove(&worker_id));
                         }
                         debug!("collected initial barrier");
-                        Some(new_epoch)
+                        Some(barrier_info.curr_epoch)
                     } else {
                         assert!(info.is_empty());
                         None
                     };
 
                     (
-                        BarrierManagerState::new(new_epoch, info, subscription_info, paused_reason),
+                        BarrierWorkerState::new(new_epoch, info, subscription_info, paused_reason),
                         active_streaming_nodes,
                         control_stream_manager,
-                        tracker,
+                        CreateMviewProgressTracker::recover(
+                            background_mview_progress,
+                            &version_stats,
+                        ),
+                        version_stats,
                     )
-                };
-                if recovery_result.is_err() {
-                    self.context.metrics.recovery_failure_cnt.inc();
                 }
-                recovery_result
+            };
+            if recovery_result.is_err() {
+                GLOBAL_META_METRICS.recovery_failure_cnt.inc();
             }
-            .instrument(tracing::info_span!("recovery_attempt"))
+            recovery_result
         })
+        .instrument(tracing::info_span!("recovery_attempt"))
         .await
         .expect("Retry until recovery success.");
 
         recovery_timer.observe_duration();
-        self.scheduled_barriers.mark_ready();
 
         let create_mview_tracker: CreateMviewProgressTracker;
-
+        let state: BarrierWorkerState;
+        let version_stats: HummockVersionStats;
         (
-            self.state,
+            state,
             self.active_streaming_nodes,
             self.control_stream_manager,
             create_mview_tracker,
+            version_stats,
         ) = new_state;
 
-        self.checkpoint_control =
-            CheckpointControl::new(self.context.clone(), create_mview_tracker).await;
-
         tracing::info!(
-            epoch = self.state.in_flight_prev_epoch().map(|epoch| epoch.value().0),
-            paused = ?self.state.paused_reason(),
+            epoch = state.in_flight_prev_epoch().map(|epoch| epoch.value().0),
+            paused = ?state.paused_reason(),
             "recovery success"
         );
 
+        self.checkpoint_control =
+            CheckpointControl::new(create_mview_tracker, state, version_stats);
+
         self.env
             .notification_manager()
             .notify_frontend_without_version(Operation::Update, Info::Recovery(Recovery {}));
     }
 }
 
-impl GlobalBarrierManagerContext {
+impl GlobalBarrierWorkerContextImpl {
+    // Migration timeout.
+    const RECOVERY_FORCE_MIGRATION_TIMEOUT: Duration = Duration::from_secs(300);
+
     /// Migrate actors in expired CNs to newly joined ones, return true if any actor is migrated.
     async fn migrate_actors(
         &self,
@@ -445,7 +493,7 @@ impl GlobalBarrierManagerContext {
             let mut available_size = new_worker_slots.len();
 
             if available_size < to_migration_size
-                && start.elapsed() > GlobalBarrierManager::RECOVERY_FORCE_MIGRATION_TIMEOUT
+                && start.elapsed() > Self::RECOVERY_FORCE_MIGRATION_TIMEOUT
             {
                 let mut factor = 2;
 
@@ -505,7 +553,7 @@ impl GlobalBarrierManagerContext {
             let changed = active_nodes
                 .wait_changed(
                     Duration::from_millis(5000),
-                    GlobalBarrierManager::RECOVERY_FORCE_MIGRATION_TIMEOUT,
+                    Self::RECOVERY_FORCE_MIGRATION_TIMEOUT,
                     |active_nodes| {
                         let current_nodes = active_nodes
                             .current()
@@ -540,7 +588,7 @@ impl GlobalBarrierManagerContext {
                 info!("integrity check passed");
             }
             Err(_) => {
-                error!("integrity check failed");
+                bail!("integrity check failed");
             }
         }
 
@@ -708,7 +756,8 @@ impl GlobalBarrierManagerContext {
         info: &InflightGraphInfo,
         active_nodes: &ActiveStreamingWorkerNodes,
     ) -> MetaResult<HashMap<WorkerId, Vec<StreamActor>>> {
-        if info.actor_map.is_empty() {
+        let actor_map = InflightFragmentInfo::actor_ids_to_collect(info.fragment_infos());
+        if actor_map.is_empty() {
             tracing::debug!("no actor to update, skipping.");
             return Ok(HashMap::new());
         }
@@ -718,8 +767,7 @@ impl GlobalBarrierManagerContext {
         // Check if any actors were dropped after info resolved.
         if all_node_actors.iter().any(|(node_id, node_actors)| {
             !active_nodes.current().contains_key(node_id)
-                || info
-                    .actor_map
+                || actor_map
                     .get(node_id)
                     .map(|actors| actors.len() != node_actors.len())
                     .unwrap_or(true)
@@ -728,7 +776,7 @@ impl GlobalBarrierManagerContext {
         }
 
         {
-            for (node_id, actors) in &info.actor_map {
+            for (node_id, actors) in &actor_map {
                 if !actors.is_empty() && !all_node_actors.contains_key(node_id) {
                     return Err(anyhow!("streaming job dropped during update").into());
                 }
@@ -749,7 +797,7 @@ mod tests {
         // total 10, assigned custom, actual 5, default full -> fixed(5)
         assert_eq!(
             TableParallelism::Fixed(5),
-            GlobalBarrierManagerContext::derive_target_parallelism(
+            GlobalBarrierWorkerContextImpl::derive_target_parallelism(
                 10,
                 TableParallelism::Custom,
                 Some(5),
@@ -760,7 +808,7 @@ mod tests {
         // total 10, assigned custom, actual 10, default full -> adaptive
         assert_eq!(
             TableParallelism::Adaptive,
-            GlobalBarrierManagerContext::derive_target_parallelism(
+            GlobalBarrierWorkerContextImpl::derive_target_parallelism(
                 10,
                 TableParallelism::Custom,
                 Some(10),
@@ -771,7 +819,7 @@ mod tests {
         // total 10, assigned custom, actual 11, default full -> adaptive
         assert_eq!(
             TableParallelism::Adaptive,
-            GlobalBarrierManagerContext::derive_target_parallelism(
+            GlobalBarrierWorkerContextImpl::derive_target_parallelism(
                 10,
                 TableParallelism::Custom,
                 Some(11),
@@ -782,7 +830,7 @@ mod tests {
         // total 10, assigned fixed(5), actual _, default full -> fixed(5)
         assert_eq!(
             TableParallelism::Adaptive,
-            GlobalBarrierManagerContext::derive_target_parallelism(
+            GlobalBarrierWorkerContextImpl::derive_target_parallelism(
                 10,
                 TableParallelism::Custom,
                 None,
@@ -793,7 +841,7 @@ mod tests {
         // total 10, assigned adaptive, actual _, default full -> adaptive
         assert_eq!(
             TableParallelism::Adaptive,
-            GlobalBarrierManagerContext::derive_target_parallelism(
+            GlobalBarrierWorkerContextImpl::derive_target_parallelism(
                 10,
                 TableParallelism::Adaptive,
                 None,
@@ -804,7 +852,7 @@ mod tests {
         // total 10, assigned adaptive, actual 5, default 5 -> fixed(5)
         assert_eq!(
             TableParallelism::Fixed(5),
-            GlobalBarrierManagerContext::derive_target_parallelism(
+            GlobalBarrierWorkerContextImpl::derive_target_parallelism(
                 10,
                 TableParallelism::Adaptive,
                 Some(5),
@@ -815,7 +863,7 @@ mod tests {
         // total 10, assigned adaptive, actual 6, default 5 -> adaptive
         assert_eq!(
             TableParallelism::Adaptive,
-            GlobalBarrierManagerContext::derive_target_parallelism(
+            GlobalBarrierWorkerContextImpl::derive_target_parallelism(
                 10,
                 TableParallelism::Adaptive,
                 Some(6),
diff --git a/src/meta/src/barrier/rpc.rs b/src/meta/src/barrier/rpc.rs
index 6e608489177be..208e108ee6c9b 100644
--- a/src/meta/src/barrier/rpc.rs
+++ b/src/meta/src/barrier/rpc.rs
@@ -25,9 +25,9 @@ use futures::StreamExt;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::tracing::TracingContext;
-use risingwave_hummock_sdk::HummockVersionId;
 use risingwave_meta_model::WorkerId;
 use risingwave_pb::common::{ActorInfo, WorkerNode};
+use risingwave_pb::meta::PausedReason;
 use risingwave_pb::stream_plan::barrier_mutation::Mutation;
 use risingwave_pb::stream_plan::{Barrier, BarrierMutation, StreamActor, SubscriptionUpstreamInfo};
 use risingwave_pb::stream_service::streaming_control_stream_request::RemovePartialGraphRequest;
@@ -43,9 +43,13 @@ use tokio_retry::strategy::ExponentialBackoff;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-use super::command::CommandContext;
-use super::{BarrierKind, GlobalBarrierManagerContext, InflightSubscriptionInfo, TracedEpoch};
-use crate::barrier::info::InflightGraphInfo;
+use super::{
+    Command, GlobalBarrierWorker, GlobalBarrierWorkerContext, GlobalBarrierWorkerContextImpl,
+    InflightSubscriptionInfo,
+};
+use crate::barrier::info::{BarrierInfo, InflightGraphInfo};
+use crate::controller::fragment::InflightFragmentInfo;
+use crate::manager::MetaSrvEnv;
 use crate::{MetaError, MetaResult};
 
 const COLLECT_ERROR_TIMEOUT: Duration = Duration::from_secs(3);
@@ -56,15 +60,15 @@ struct ControlStreamNode {
 }
 
 pub(super) struct ControlStreamManager {
-    context: GlobalBarrierManagerContext,
     nodes: HashMap<WorkerId, ControlStreamNode>,
+    env: MetaSrvEnv,
 }
 
 impl ControlStreamManager {
-    pub(super) fn new(context: GlobalBarrierManagerContext) -> Self {
+    pub(super) fn new(env: MetaSrvEnv) -> Self {
         Self {
-            context,
             nodes: Default::default(),
+            env,
         }
     }
 
@@ -72,34 +76,34 @@ impl ControlStreamManager {
         &mut self,
         node: WorkerNode,
         subscription: &InflightSubscriptionInfo,
+        context: &impl GlobalBarrierWorkerContext,
     ) {
         let node_id = node.id as WorkerId;
         if self.nodes.contains_key(&node_id) {
             warn!(id = node.id, host = ?node.host, "node already exists");
             return;
         }
-        let version_id = self
-            .context
-            .hummock_manager
-            .on_current_version(|version| version.id)
-            .await;
         let node_host = node.host.clone().unwrap();
         let mut backoff = ExponentialBackoff::from_millis(100)
             .max_delay(Duration::from_secs(3))
             .factor(5);
         const MAX_RETRY: usize = 5;
         for i in 1..=MAX_RETRY {
-            match self
-                .context
-                .new_control_stream_node(
-                    node.clone(),
-                    version_id,
-                    &subscription.mv_depended_subscriptions,
-                )
+            match context
+                .new_control_stream(&node, &subscription.mv_depended_subscriptions)
                 .await
             {
-                Ok(stream_node) => {
-                    assert!(self.nodes.insert(node_id, stream_node).is_none());
+                Ok(handle) => {
+                    assert!(self
+                        .nodes
+                        .insert(
+                            node_id,
+                            ControlStreamNode {
+                                worker: node.clone(),
+                                handle,
+                            }
+                        )
+                        .is_none());
                     info!(?node_host, "add control stream worker");
                     return;
                 }
@@ -117,20 +121,21 @@ impl ControlStreamManager {
 
     pub(super) async fn reset(
         &mut self,
-        version_id: HummockVersionId,
         subscriptions: &InflightSubscriptionInfo,
         nodes: &HashMap<WorkerId, WorkerNode>,
+        context: &impl GlobalBarrierWorkerContext,
     ) -> MetaResult<()> {
-        let nodes = try_join_all(nodes.iter().map(|(worker_id, node)| async {
-            let node = self
-                .context
-                .new_control_stream_node(
-                    node.clone(),
-                    version_id,
-                    &subscriptions.mv_depended_subscriptions,
-                )
+        let nodes = try_join_all(nodes.iter().map(|(worker_id, node)| async move {
+            let handle = context
+                .new_control_stream(node, &subscriptions.mv_depended_subscriptions)
                 .await?;
-            Result::<_, MetaError>::Ok((*worker_id, node))
+            Result::<_, MetaError>::Ok((
+                *worker_id,
+                ControlStreamNode {
+                    worker: node.clone(),
+                    handle,
+                },
+            ))
         }))
         .await?;
         self.nodes.clear();
@@ -143,7 +148,7 @@ impl ControlStreamManager {
 
     /// Clear all nodes and response streams in the manager.
     pub(super) fn clear(&mut self) {
-        *self = Self::new(self.context.clone());
+        *self = Self::new(self.env.clone());
     }
 
     async fn next_response(
@@ -203,7 +208,7 @@ impl ControlStreamManager {
         Some((worker_id, result))
     }
 
-    pub(super) async fn next_complete_barrier_response(
+    pub(super) async fn next_collect_barrier_response(
         &mut self,
     ) -> (WorkerId, MetaResult<BarrierCompleteResponse>) {
         use streaming_control_stream_response::Response;
@@ -248,11 +253,13 @@ impl ControlStreamManager {
 impl ControlStreamManager {
     pub(super) fn inject_command_ctx_barrier(
         &mut self,
-        command_ctx: &CommandContext,
+        command: Option<&Command>,
+        barrier_info: &BarrierInfo,
+        prev_paused_reason: Option<PausedReason>,
         pre_applied_graph_info: &InflightGraphInfo,
-        applied_graph_info: Option<&InflightGraphInfo>,
+        applied_graph_info: &InflightGraphInfo,
     ) -> MetaResult<HashSet<WorkerId>> {
-        let mutation = command_ctx.to_mutation();
+        let mutation = command.and_then(|c| c.to_mutation(prev_paused_reason));
         let subscriptions_to_add = if let Some(Mutation::Add(add)) = &mutation {
             add.subscriptions_to_add.clone()
         } else {
@@ -266,24 +273,25 @@ impl ControlStreamManager {
         self.inject_barrier(
             None,
             mutation,
-            (&command_ctx.curr_epoch, &command_ctx.prev_epoch),
-            &command_ctx.kind,
-            pre_applied_graph_info,
-            applied_graph_info,
-            command_ctx.command.actors_to_create(),
+            barrier_info,
+            pre_applied_graph_info.fragment_infos(),
+            applied_graph_info.fragment_infos(),
+            command
+                .as_ref()
+                .map(|command| command.actors_to_create())
+                .unwrap_or_default(),
             subscriptions_to_add,
             subscriptions_to_remove,
         )
     }
 
-    pub(super) fn inject_barrier(
+    pub(super) fn inject_barrier<'a>(
         &mut self,
         creating_table_id: Option<TableId>,
         mutation: Option<Mutation>,
-        (curr_epoch, prev_epoch): (&TracedEpoch, &TracedEpoch),
-        kind: &BarrierKind,
-        pre_applied_graph_info: &InflightGraphInfo,
-        applied_graph_info: Option<&InflightGraphInfo>,
+        barrier_info: &BarrierInfo,
+        pre_applied_graph_info: impl IntoIterator<Item = &InflightFragmentInfo>,
+        applied_graph_info: impl IntoIterator<Item = &'a InflightFragmentInfo> + 'a,
         mut new_actors: Option<HashMap<WorkerId, Vec<StreamActor>>>,
         subscriptions_to_add: Vec<SubscriptionUpstreamInfo>,
         subscriptions_to_remove: Vec<SubscriptionUpstreamInfo>,
@@ -296,24 +304,19 @@ impl ControlStreamManager {
             .map(|table_id| table_id.table_id)
             .unwrap_or(u32::MAX);
 
-        for worker_id in pre_applied_graph_info
-            .worker_ids()
-            .chain(
-                applied_graph_info
-                    .into_iter()
-                    .flat_map(|info| info.worker_ids()),
-            )
-            .chain(
-                new_actors
-                    .iter()
-                    .flat_map(|new_actors| new_actors.keys().cloned()),
-            )
-        {
-            if !self.nodes.contains_key(&(worker_id as _)) {
+        let node_actors = InflightFragmentInfo::actor_ids_to_collect(pre_applied_graph_info);
+
+        for worker_id in node_actors.keys() {
+            if !self.nodes.contains_key(worker_id) {
                 return Err(anyhow!("unconnected worker node {}", worker_id).into());
             }
         }
 
+        let table_ids_to_sync: HashSet<_> =
+            InflightFragmentInfo::existing_table_ids(applied_graph_info)
+                .map(|table_id| table_id.table_id)
+                .collect();
+
         let mut node_need_collect = HashSet::new();
         let new_actors_location_to_broadcast = new_actors
             .iter()
@@ -323,7 +326,7 @@ impl ControlStreamManager {
                     actor_id: actor_info.actor_id,
                     host: self
                         .nodes
-                        .get(&(*worker_id as _))
+                        .get(worker_id)
                         .expect("have checked exist previously")
                         .worker
                         .host
@@ -335,28 +338,23 @@ impl ControlStreamManager {
         self.nodes
             .iter()
             .try_for_each(|(node_id, node)| {
-                let actor_ids_to_collect: Vec<_> = pre_applied_graph_info
-                    .actor_ids_to_collect(*node_id as _)
+                let actor_ids_to_collect = node_actors
+                    .get(node_id)
+                    .map(|actors| actors.iter().cloned())
+                    .into_iter()
+                    .flatten()
                     .collect();
-                let table_ids_to_sync = if let Some(graph_info) = applied_graph_info {
-                    graph_info
-                        .existing_table_ids()
-                        .map(|table_id| table_id.table_id)
-                        .collect()
-                } else {
-                    Default::default()
-                };
-
                 {
                     let mutation = mutation.clone();
                     let barrier = Barrier {
                         epoch: Some(risingwave_pb::data::Epoch {
-                            curr: curr_epoch.value().0,
-                            prev: prev_epoch.value().0,
+                            curr: barrier_info.curr_epoch.value().0,
+                            prev: barrier_info.prev_epoch(),
                         }),
                         mutation: mutation.clone().map(|_| BarrierMutation { mutation }),
-                        tracing_context: TracingContext::from_span(curr_epoch.span()).to_protobuf(),
-                        kind: kind.to_protobuf() as i32,
+                        tracing_context: TracingContext::from_span(barrier_info.curr_epoch.span())
+                            .to_protobuf(),
+                        kind: barrier_info.kind.to_protobuf() as i32,
                         passed_actors: vec![],
                     };
 
@@ -369,7 +367,10 @@ impl ControlStreamManager {
                                         request_id: Uuid::new_v4().to_string(),
                                         barrier: Some(barrier),
                                         actor_ids_to_collect,
-                                        table_ids_to_sync,
+                                        table_ids_to_sync: table_ids_to_sync
+                                            .iter()
+                                            .cloned()
+                                            .collect(),
                                         partial_graph_id,
                                         broadcast_info: new_actors_location_to_broadcast.clone(),
                                         actors_to_build: new_actors
@@ -401,12 +402,11 @@ impl ControlStreamManager {
                 // Record failure in event log.
                 use risingwave_pb::meta::event_log;
                 let event = event_log::EventInjectBarrierFail {
-                    prev_epoch: prev_epoch.value().0,
-                    cur_epoch: curr_epoch.value().0,
+                    prev_epoch: barrier_info.prev_epoch(),
+                    cur_epoch: barrier_info.curr_epoch.value().0,
                     error: e.to_report_string(),
                 };
-                self.context
-                    .env
+                self.env
                     .event_log_manager_ref()
                     .add_event_logs(vec![event_log::Event::InjectBarrierFail(event)]);
             })?;
@@ -434,37 +434,35 @@ impl ControlStreamManager {
     }
 }
 
-impl GlobalBarrierManagerContext {
-    async fn new_control_stream_node(
+impl GlobalBarrierWorkerContextImpl {
+    pub(super) async fn new_control_stream_impl(
         &self,
-        node: WorkerNode,
-        initial_version_id: HummockVersionId,
+        node: &WorkerNode,
         mv_depended_subscriptions: &HashMap<TableId, HashMap<u32, u64>>,
-    ) -> MetaResult<ControlStreamNode> {
+    ) -> MetaResult<StreamingControlHandle> {
+        let initial_version_id = self
+            .hummock_manager
+            .on_current_version(|version| version.id)
+            .await;
         let handle = self
             .env
             .stream_client_pool()
-            .get(&node)
+            .get(node)
             .await?
             .start_streaming_control(initial_version_id, mv_depended_subscriptions)
             .await?;
-        Ok(ControlStreamNode {
-            worker: node.clone(),
-            handle,
-        })
+        Ok(handle)
     }
+}
 
+impl<C> GlobalBarrierWorker<C> {
     /// Send barrier-complete-rpc and wait for responses from all CNs
-    pub(super) fn report_collect_failure(
-        &self,
-        command_context: &CommandContext,
-        error: &MetaError,
-    ) {
+    pub(super) fn report_collect_failure(&self, barrier_info: &BarrierInfo, error: &MetaError) {
         // Record failure in event log.
         use risingwave_pb::meta::event_log;
         let event = event_log::EventCollectBarrierFail {
-            prev_epoch: command_context.prev_epoch.value().0,
-            cur_epoch: command_context.curr_epoch.value().0,
+            prev_epoch: barrier_info.prev_epoch(),
+            cur_epoch: barrier_info.curr_epoch.value().0,
             error: error.to_report_string(),
         };
         self.env
diff --git a/src/meta/src/barrier/schedule.rs b/src/meta/src/barrier/schedule.rs
index c39fdb56e4fb7..ee4c4133f9287 100644
--- a/src/meta/src/barrier/schedule.rs
+++ b/src/meta/src/barrier/schedule.rs
@@ -20,7 +20,8 @@ use std::time::{Duration, Instant};
 use anyhow::{anyhow, Context};
 use assert_matches::assert_matches;
 use parking_lot::Mutex;
-use risingwave_common::catalog::TableId;
+use prometheus::HistogramTimer;
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_hummock_sdk::HummockVersionId;
 use risingwave_pb::meta::PausedReason;
 use tokio::select;
@@ -28,7 +29,7 @@ use tokio::sync::{oneshot, watch};
 use tokio::time::Interval;
 
 use super::notifier::Notifier;
-use super::{Command, Scheduled};
+use super::{Command, GlobalBarrierWorkerContext, NewBarrier, Scheduled};
 use crate::hummock::HummockManagerRef;
 use crate::model::ActorId;
 use crate::rpc::metrics::MetaMetrics;
@@ -56,8 +57,16 @@ enum QueueStatus {
     Blocked(String),
 }
 
+struct ScheduledQueueItem {
+    database_id: DatabaseId,
+    command: Command,
+    notifiers: Vec<Notifier>,
+    send_latency_timer: HistogramTimer,
+    span: tracing::Span,
+}
+
 pub(super) struct ScheduledQueue {
-    queue: VecDeque<Scheduled>,
+    queue: VecDeque<ScheduledQueueItem>,
     status: QueueStatus,
 }
 
@@ -81,7 +90,7 @@ impl ScheduledQueue {
         self.queue.len()
     }
 
-    fn push_back(&mut self, scheduled: Scheduled) -> MetaResult<()> {
+    fn push_back(&mut self, scheduled: ScheduledQueueItem) -> MetaResult<()> {
         // We don't allow any command to be scheduled when the queue is blocked, except for dropping streaming jobs.
         // Because we allow dropping streaming jobs when the cluster is under recovery, so we have to buffer the drop
         // command and execute it when the cluster is ready to clean up it.
@@ -100,27 +109,35 @@ impl ScheduledQueue {
     }
 }
 
+fn tracing_span() -> tracing::Span {
+    if tracing::Span::current().is_none() {
+        tracing::Span::none()
+    } else {
+        tracing::info_span!(
+            "barrier",
+            checkpoint = tracing::field::Empty,
+            epoch = tracing::field::Empty
+        )
+    }
+}
+
 impl Inner {
     /// Create a new scheduled barrier with the given `checkpoint`, `command` and `notifiers`.
     fn new_scheduled(
         &self,
-        checkpoint: bool,
+        database_id: DatabaseId,
         command: Command,
         notifiers: impl IntoIterator<Item = Notifier>,
-    ) -> Scheduled {
+    ) -> ScheduledQueueItem {
         // Create a span only if we're being traced, instead of for every periodic barrier.
-        let span = if tracing::Span::current().is_none() {
-            tracing::Span::none()
-        } else {
-            tracing::info_span!("barrier", checkpoint, epoch = tracing::field::Empty)
-        };
+        let span = tracing_span();
 
-        Scheduled {
+        ScheduledQueueItem {
+            database_id,
             command,
             notifiers: notifiers.into_iter().collect(),
             send_latency_timer: self.metrics.barrier_send_latency.start_timer(),
             span,
-            checkpoint,
         }
     }
 }
@@ -141,12 +158,7 @@ impl BarrierScheduler {
     pub fn new_pair(
         hummock_manager: HummockManagerRef,
         metrics: Arc<MetaMetrics>,
-        checkpoint_frequency: usize,
     ) -> (Self, ScheduledBarriers) {
-        tracing::info!(
-            "Starting barrier scheduler with: checkpoint_frequency={:?}",
-            checkpoint_frequency,
-        );
         let inner = Arc::new(Inner {
             queue: Mutex::new(ScheduledQueue::new()),
             changed_tx: watch::channel(()).0,
@@ -158,18 +170,12 @@ impl BarrierScheduler {
                 inner: inner.clone(),
                 hummock_manager,
             },
-            ScheduledBarriers {
-                num_uncheckpointed_barrier: 0,
-                force_checkpoint: false,
-                checkpoint_frequency,
-                inner,
-                min_interval: None,
-            },
+            ScheduledBarriers { inner },
         )
     }
 
     /// Push a scheduled barrier into the queue.
-    fn push(&self, scheduleds: impl IntoIterator<Item = Scheduled>) -> MetaResult<()> {
+    fn push(&self, scheduleds: impl IntoIterator<Item = ScheduledQueueItem>) -> MetaResult<()> {
         let mut queue = self.inner.queue.lock();
         for scheduled in scheduleds {
             queue.push_back(scheduled)?;
@@ -200,58 +206,17 @@ impl BarrierScheduler {
         }
     }
 
-    /// Attach `new_notifiers` to the very first scheduled barrier. If there's no one scheduled, a
-    /// default barrier will be created. If `new_checkpoint` is true, the barrier will become a
-    /// checkpoint.
-    fn attach_notifiers(
-        &self,
-        new_notifiers: Vec<Notifier>,
-        new_checkpoint: bool,
-    ) -> MetaResult<()> {
-        let mut queue = self.inner.queue.lock();
-        match queue.queue.front_mut() {
-            Some(Scheduled {
-                notifiers,
-                checkpoint,
-                ..
-            }) => {
-                notifiers.extend(new_notifiers);
-                *checkpoint = *checkpoint || new_checkpoint;
-            }
-            None => {
-                // If no command scheduled, create a periodic barrier by default.
-                queue.push_back(self.inner.new_scheduled(
-                    new_checkpoint,
-                    Command::barrier(),
-                    new_notifiers,
-                ))?;
-                self.inner.changed_tx.send(()).ok();
-            }
-        }
-        Ok(())
-    }
-
-    /// Wait for the next barrier to collect. Note that the barrier flowing in our stream graph is
-    /// ignored, if exists.
-    pub async fn wait_for_next_barrier_to_collect(&self, checkpoint: bool) -> MetaResult<()> {
-        let (tx, rx) = oneshot::channel();
-        let notifier = Notifier {
-            collected: Some(tx),
-            ..Default::default()
-        };
-        self.attach_notifiers(vec![notifier], checkpoint)?;
-        rx.await
-            .ok()
-            .context("failed to wait for barrier collect")?
-    }
-
     /// Run multiple commands and return when they're all completely finished. It's ensured that
     /// multiple commands are executed continuously.
     ///
     /// Returns the barrier info of each command.
     ///
     /// TODO: atomicity of multiple commands is not guaranteed.
-    async fn run_multiple_commands(&self, commands: Vec<Command>) -> MetaResult<()> {
+    async fn run_multiple_commands(
+        &self,
+        database_id: DatabaseId,
+        commands: Vec<Command>,
+    ) -> MetaResult<()> {
         let mut contexts = Vec::with_capacity(commands.len());
         let mut scheduleds = Vec::with_capacity(commands.len());
 
@@ -261,7 +226,7 @@ impl BarrierScheduler {
 
             contexts.push((started_rx, collect_rx));
             scheduleds.push(self.inner.new_scheduled(
-                command.need_checkpoint(),
+                database_id,
                 command,
                 once(Notifier {
                     started: Some(started_tx),
@@ -295,31 +260,39 @@ impl BarrierScheduler {
     /// configuration change.
     ///
     /// Returns the barrier info of the actual command.
-    pub async fn run_config_change_command_with_pause(&self, command: Command) -> MetaResult<()> {
-        self.run_multiple_commands(vec![
-            Command::pause(PausedReason::ConfigChange),
-            command,
-            Command::resume(PausedReason::ConfigChange),
-        ])
+    pub async fn run_config_change_command_with_pause(
+        &self,
+        database_id: DatabaseId,
+        command: Command,
+    ) -> MetaResult<()> {
+        self.run_multiple_commands(
+            database_id,
+            vec![
+                Command::pause(PausedReason::ConfigChange),
+                command,
+                Command::resume(PausedReason::ConfigChange),
+            ],
+        )
         .await
     }
 
     /// Run a command and return when it's completely finished.
     ///
     /// Returns the barrier info of the actual command.
-    pub async fn run_command(&self, command: Command) -> MetaResult<()> {
+    pub async fn run_command(&self, database_id: DatabaseId, command: Command) -> MetaResult<()> {
         tracing::trace!("run_command: {:?}", command);
-        let ret = self.run_multiple_commands(vec![command]).await;
+        let ret = self.run_multiple_commands(database_id, vec![command]).await;
         tracing::trace!("run_command finished");
         ret
     }
 
     /// Flush means waiting for the next barrier to collect.
-    pub async fn flush(&self, checkpoint: bool) -> MetaResult<HummockVersionId> {
+    pub async fn flush(&self, database_id: DatabaseId) -> MetaResult<HummockVersionId> {
         let start = Instant::now();
 
         tracing::debug!("start barrier flush");
-        self.wait_for_next_barrier_to_collect(checkpoint).await?;
+        self.run_multiple_commands(database_id, vec![Command::Flush])
+            .await?;
 
         let elapsed = Instant::now().duration_since(start);
         tracing::debug!("barrier flushed in {:?}", elapsed);
@@ -330,9 +303,13 @@ impl BarrierScheduler {
 }
 
 /// The receiver side of the barrier scheduling queue.
-/// Held by the [`super::GlobalBarrierManager`] to execute these commands.
 pub struct ScheduledBarriers {
-    min_interval: Option<Interval>,
+    inner: Arc<Inner>,
+}
+
+/// Held by the [`super::GlobalBarrierWorker`] to execute these commands.
+pub(super) struct PeriodicBarriers {
+    min_interval: Interval,
 
     /// Force checkpoint in next barrier.
     force_checkpoint: bool,
@@ -340,36 +317,49 @@ pub struct ScheduledBarriers {
     /// The numbers of barrier (checkpoint = false) since the last barrier (checkpoint = true)
     num_uncheckpointed_barrier: usize,
     checkpoint_frequency: usize,
-    inner: Arc<Inner>,
 }
 
-impl ScheduledBarriers {
+impl PeriodicBarriers {
+    pub(super) fn new(min_interval: Duration, checkpoint_frequency: usize) -> PeriodicBarriers {
+        Self {
+            min_interval: tokio::time::interval(min_interval),
+            force_checkpoint: false,
+            num_uncheckpointed_barrier: 0,
+            checkpoint_frequency,
+        }
+    }
+
     pub(super) fn set_min_interval(&mut self, min_interval: Duration) {
-        let set_new_interval = match &self.min_interval {
-            None => true,
-            Some(prev_min_interval) => min_interval != prev_min_interval.period(),
-        };
+        let set_new_interval = min_interval != self.min_interval.period();
         if set_new_interval {
             let mut min_interval = tokio::time::interval(min_interval);
             min_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-            self.min_interval = Some(min_interval);
+            self.min_interval = min_interval;
         }
     }
 
-    pub(super) async fn next_barrier(&mut self) -> Scheduled {
+    pub(super) async fn next_barrier(
+        &mut self,
+        context: &impl GlobalBarrierWorkerContext,
+    ) -> NewBarrier {
         let checkpoint = self.try_get_checkpoint();
         let scheduled = select! {
             biased;
-            mut scheduled = self.inner.next_scheduled() => {
-                if let Some(min_interval) = &mut self.min_interval {
-                    min_interval.reset();
+            scheduled = context.next_scheduled() => {
+                self.min_interval.reset();
+                let checkpoint = scheduled.command.need_checkpoint() || checkpoint;
+                NewBarrier {
+                    command: Some((scheduled.database_id, scheduled.command, scheduled.notifiers)),
+                    span: scheduled.span,
+                    checkpoint,
                 }
-                scheduled.checkpoint = scheduled.checkpoint || checkpoint;
-                scheduled
             },
-            _ = self.min_interval.as_mut().expect("should have set min interval").tick() => {
-                self.inner
-                    .new_scheduled(checkpoint, Command::barrier(), std::iter::empty())
+            _ = self.min_interval.tick() => {
+                NewBarrier {
+                    command: None,
+                    span: tracing_span(),
+                    checkpoint,
+                }
             }
         };
         self.update_num_uncheckpointed_barrier(scheduled.checkpoint);
@@ -377,14 +367,20 @@ impl ScheduledBarriers {
     }
 }
 
-impl Inner {
-    async fn next_scheduled(&self) -> Scheduled {
+impl ScheduledBarriers {
+    pub(super) async fn next_scheduled(&self) -> Scheduled {
         loop {
-            let mut rx = self.changed_tx.subscribe();
+            let mut rx = self.inner.changed_tx.subscribe();
             {
-                let mut queue = self.queue.lock();
-                if let Some(scheduled) = queue.queue.pop_front() {
-                    break scheduled;
+                let mut queue = self.inner.queue.lock();
+                if let Some(item) = queue.queue.pop_front() {
+                    item.send_latency_timer.observe_duration();
+                    break Scheduled {
+                        database_id: item.database_id,
+                        command: item.command,
+                        notifiers: item.notifiers,
+                        span: item.span,
+                    };
                 }
             }
             rx.changed().await.unwrap();
@@ -398,7 +394,7 @@ impl ScheduledBarriers {
     pub(super) fn abort_and_mark_blocked(&self, reason: impl Into<String> + Copy) {
         let mut queue = self.inner.queue.lock();
         queue.mark_blocked(reason.into());
-        while let Some(Scheduled { notifiers, .. }) = queue.queue.pop_front() {
+        while let Some(ScheduledQueueItem { notifiers, .. }) = queue.queue.pop_front() {
             notifiers
                 .into_iter()
                 .for_each(|notify| notify.notify_collection_failed(anyhow!(reason.into()).into()))
@@ -418,7 +414,7 @@ impl ScheduledBarriers {
         assert_matches!(queue.status, QueueStatus::Blocked(_));
         let (mut dropped_actors, mut cancel_table_ids) = (vec![], HashSet::new());
 
-        while let Some(Scheduled {
+        while let Some(ScheduledQueueItem {
             notifiers, command, ..
         }) = queue.queue.pop_front()
         {
@@ -440,7 +436,9 @@ impl ScheduledBarriers {
         }
         (dropped_actors, cancel_table_ids)
     }
+}
 
+impl PeriodicBarriers {
     /// Whether the barrier(checkpoint = true) should be injected.
     fn try_get_checkpoint(&self) -> bool {
         self.num_uncheckpointed_barrier + 1 >= self.checkpoint_frequency || self.force_checkpoint
diff --git a/src/meta/src/barrier/state.rs b/src/meta/src/barrier/state.rs
index db2ded5629d7a..1df9f0033e942 100644
--- a/src/meta/src/barrier/state.rs
+++ b/src/meta/src/barrier/state.rs
@@ -12,20 +12,28 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashSet;
+use std::mem::take;
+
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_common::util::epoch::Epoch;
 use risingwave_pb::meta::PausedReason;
 
-use crate::barrier::info::{InflightGraphInfo, InflightSubscriptionInfo};
-use crate::barrier::{Command, CreateStreamingJobType, TracedEpoch};
+use crate::barrier::info::{BarrierInfo, InflightGraphInfo, InflightSubscriptionInfo};
+use crate::barrier::{BarrierKind, Command, CreateStreamingJobType, TracedEpoch};
+use crate::controller::fragment::InflightFragmentInfo;
 
-/// `BarrierManagerState` defines the necessary state of `GlobalBarrierManager`.
-pub struct BarrierManagerState {
+/// The latest state of `GlobalBarrierWorker` after injecting the latest barrier.
+pub(super) struct BarrierWorkerState {
     /// The last sent `prev_epoch`
     ///
     /// There's no need to persist this field. On recovery, we will restore this from the latest
     /// committed snapshot in `HummockManager`.
     in_flight_prev_epoch: Option<TracedEpoch>,
 
+    /// The `prev_epoch` of pending non checkpoint barriers
+    pending_non_checkpoint_barriers: Vec<u64>,
+
     /// Inflight running actors info.
     pub(crate) inflight_graph_info: InflightGraphInfo,
 
@@ -35,7 +43,7 @@ pub struct BarrierManagerState {
     paused_reason: Option<PausedReason>,
 }
 
-impl BarrierManagerState {
+impl BarrierWorkerState {
     pub fn new(
         in_flight_prev_epoch: Option<TracedEpoch>,
         inflight_graph_info: InflightGraphInfo,
@@ -44,6 +52,7 @@ impl BarrierManagerState {
     ) -> Self {
         Self {
             in_flight_prev_epoch,
+            pending_non_checkpoint_barriers: vec![],
             inflight_graph_info,
             inflight_subscription_info,
             paused_reason,
@@ -54,7 +63,7 @@ impl BarrierManagerState {
         self.paused_reason
     }
 
-    pub fn set_paused_reason(&mut self, paused_reason: Option<PausedReason>) {
+    fn set_paused_reason(&mut self, paused_reason: Option<PausedReason>) {
         if self.paused_reason != paused_reason {
             tracing::info!(current = ?self.paused_reason, new = ?paused_reason, "update paused state");
             self.paused_reason = paused_reason;
@@ -65,10 +74,14 @@ impl BarrierManagerState {
         self.in_flight_prev_epoch.as_ref()
     }
 
-    /// Returns the epoch pair for the next barrier, and updates the state.
-    pub fn next_epoch_pair(&mut self, command: &Command) -> Option<(TracedEpoch, TracedEpoch)> {
+    /// Returns the `BarrierInfo` for the next barrier, and updates the state.
+    pub fn next_barrier_info(
+        &mut self,
+        command: Option<&Command>,
+        is_checkpoint: bool,
+    ) -> Option<BarrierInfo> {
         if self.inflight_graph_info.is_empty()
-            && !matches!(&command, Command::CreateStreamingJob { .. })
+            && !matches!(&command, Some(Command::CreateStreamingJob { .. }))
         {
             return None;
         };
@@ -76,40 +89,94 @@ impl BarrierManagerState {
             .in_flight_prev_epoch
             .get_or_insert_with(|| TracedEpoch::new(Epoch::now()));
         let prev_epoch = in_flight_prev_epoch.clone();
-        let next_epoch = prev_epoch.next();
-        *in_flight_prev_epoch = next_epoch.clone();
-        Some((prev_epoch, next_epoch))
+        let curr_epoch = prev_epoch.next();
+        *in_flight_prev_epoch = curr_epoch.clone();
+        self.pending_non_checkpoint_barriers
+            .push(prev_epoch.value().0);
+        let kind = if is_checkpoint {
+            let epochs = take(&mut self.pending_non_checkpoint_barriers);
+            BarrierKind::Checkpoint(epochs)
+        } else {
+            BarrierKind::Barrier
+        };
+        Some(BarrierInfo {
+            prev_epoch,
+            curr_epoch,
+            kind,
+        })
     }
 
     /// Returns the inflight actor infos that have included the newly added actors in the given command. The dropped actors
     /// will be removed from the state after the info get resolved.
+    ///
+    /// Return (`graph_info`, `subscription_info`, `table_ids_to_commit`, `jobs_to_wait`, `prev_paused_reason`)
     pub fn apply_command(
         &mut self,
-        command: &Command,
-    ) -> (InflightGraphInfo, InflightSubscriptionInfo) {
+        command: Option<&Command>,
+        database_id: Option<DatabaseId>,
+    ) -> (
+        InflightGraphInfo,
+        InflightSubscriptionInfo,
+        HashSet<TableId>,
+        HashSet<TableId>,
+        Option<PausedReason>,
+    ) {
         // update the fragment_infos outside pre_apply
-        let fragment_changes = if let Command::CreateStreamingJob {
+        let fragment_changes = if let Some(Command::CreateStreamingJob {
             job_type: CreateStreamingJobType::SnapshotBackfill(_),
             ..
-        } = command
+        }) = command
         {
             None
-        } else if let Some(fragment_changes) = command.fragment_changes() {
-            self.inflight_graph_info.pre_apply(&fragment_changes);
+        } else if let Some(fragment_changes) = command.and_then(Command::fragment_changes) {
+            self.inflight_graph_info.pre_apply(
+                &fragment_changes,
+                database_id.expect("should exist when having fragment changes"),
+            );
             Some(fragment_changes)
         } else {
             None
         };
-        self.inflight_subscription_info.pre_apply(command);
+        if let Some(command) = &command {
+            self.inflight_subscription_info.pre_apply(command);
+        }
 
         let info = self.inflight_graph_info.clone();
         let subscription_info = self.inflight_subscription_info.clone();
 
         if let Some(fragment_changes) = fragment_changes {
-            self.inflight_graph_info.post_apply(&fragment_changes);
+            self.inflight_graph_info.post_apply(
+                &fragment_changes,
+                database_id.expect("should exist when having fragment changes"),
+            );
+        }
+
+        let mut table_ids_to_commit: HashSet<_> = info.existing_table_ids().collect();
+        let mut jobs_to_wait = HashSet::new();
+        if let Some(Command::MergeSnapshotBackfillStreamingJobs(jobs_to_merge)) = command {
+            for (table_id, (_, graph_info)) in jobs_to_merge {
+                jobs_to_wait.insert(*table_id);
+                table_ids_to_commit.extend(InflightFragmentInfo::existing_table_ids(
+                    graph_info.fragment_infos(),
+                ));
+                self.inflight_graph_info.extend(graph_info.clone());
+            }
         }
-        self.inflight_subscription_info.post_apply(command);
 
-        (info, subscription_info)
+        if let Some(command) = command {
+            self.inflight_subscription_info.post_apply(command);
+        }
+
+        let prev_paused_reason = self.paused_reason;
+        let curr_paused_reason = Command::next_paused_reason(command, prev_paused_reason);
+        self.set_paused_reason(curr_paused_reason);
+
+        (
+            info,
+            subscription_info,
+            table_ids_to_commit,
+            jobs_to_wait,
+            prev_paused_reason,
+        )
     }
 }
diff --git a/src/meta/src/controller/catalog.rs b/src/meta/src/controller/catalog.rs
index 08824459e916e..2bdd59ea39124 100644
--- a/src/meta/src/controller/catalog.rs
+++ b/src/meta/src/controller/catalog.rs
@@ -32,9 +32,9 @@ use risingwave_meta_model::{
     actor, connection, database, fragment, function, index, object, object_dependency, schema,
     secret, sink, source, streaming_job, subscription, table, user_privilege, view, ActorId,
     ActorUpstreamActors, ColumnCatalogArray, ConnectionId, CreateType, DatabaseId, FragmentId,
-    FunctionId, I32Array, IndexId, JobStatus, ObjectId, PrivateLinkService, Property, SchemaId,
-    SecretId, SinkId, SourceId, StreamNode, StreamSourceInfo, StreamingParallelism, SubscriptionId,
-    TableId, UserId, ViewId,
+    FunctionId, I32Array, IndexId, JobStatus, ObjectId, Property, SchemaId, SecretId, SinkId,
+    SourceId, StreamNode, StreamSourceInfo, StreamingParallelism, SubscriptionId, TableId, UserId,
+    ViewId,
 };
 use risingwave_pb::catalog::subscription::SubscriptionState;
 use risingwave_pb::catalog::table::PbTableType;
@@ -57,24 +57,30 @@ use sea_orm::ActiveValue::Set;
 use sea_orm::{
     ActiveModelTrait, ColumnTrait, DatabaseConnection, DatabaseTransaction, EntityTrait,
     IntoActiveModel, JoinType, PaginatorTrait, QueryFilter, QuerySelect, RelationTrait,
-    TransactionTrait, Value,
+    SelectColumns, TransactionTrait, Value,
 };
 use tokio::sync::oneshot::Sender;
 use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 use tracing::info;
 
-use super::utils::{check_subscription_name_duplicate, get_fragment_ids_by_jobs};
-use crate::controller::rename::{alter_relation_rename, alter_relation_rename_refs};
+use super::utils::{
+    check_subscription_name_duplicate, get_fragment_ids_by_jobs, get_internal_tables_by_id,
+    rename_relation, rename_relation_refer,
+};
 use crate::controller::utils::{
-    build_relation_group, check_connection_name_duplicate, check_database_name_duplicate,
-    check_function_signature_duplicate, check_relation_name_duplicate, check_schema_name_duplicate,
-    check_secret_name_duplicate, ensure_object_id, ensure_object_not_refer, ensure_schema_empty,
-    ensure_user_id, extract_external_table_name_from_definition, get_referring_objects,
+    build_relation_group_for_delete, check_connection_name_duplicate,
+    check_database_name_duplicate, check_function_signature_duplicate,
+    check_relation_name_duplicate, check_schema_name_duplicate, check_secret_name_duplicate,
+    ensure_object_id, ensure_object_not_refer, ensure_schema_empty, ensure_user_id,
+    extract_external_table_name_from_definition, get_referring_objects,
     get_referring_objects_cascade, get_user_privilege, list_user_info_by_ids,
     resolve_source_register_info_for_jobs, PartialObject,
 };
 use crate::controller::ObjectModel;
-use crate::manager::{MetaSrvEnv, NotificationVersion, IGNORED_NOTIFICATION_VERSION};
+use crate::manager::{
+    get_referred_secret_ids_from_source, MetaSrvEnv, NotificationVersion,
+    IGNORED_NOTIFICATION_VERSION,
+};
 use crate::rpc::ddl_controller::DropMode;
 use crate::telemetry::MetaTelemetryJobDesc;
 use crate::{MetaError, MetaResult};
@@ -103,13 +109,15 @@ pub struct CatalogController {
 
 #[derive(Clone, Default)]
 pub struct ReleaseContext {
+    pub(crate) database_id: DatabaseId,
     pub(crate) streaming_job_ids: Vec<ObjectId>,
     /// Dropped state table list, need to unregister from hummock.
     pub(crate) state_table_ids: Vec<TableId>,
     /// Dropped source list, need to unregister from source manager.
     pub(crate) source_ids: Vec<SourceId>,
     /// Dropped connection list, need to delete from vpc endpoints.
-    pub(crate) connections: Vec<PrivateLinkService>,
+    #[allow(dead_code)]
+    pub(crate) connections: Vec<ConnectionId>,
 
     /// Dropped fragments that are fetching data from the target source.
     pub(crate) source_fragments: HashMap<SourceId, BTreeSet<FragmentId>>,
@@ -367,7 +375,7 @@ impl CatalogController {
             .all(&txn)
             .await?
             .into_iter()
-            .map(|conn| conn.info)
+            .map(|conn| conn.connection_id)
             .collect_vec();
 
         // Find affect users with privileges on the database and the objects in the database.
@@ -418,6 +426,7 @@ impl CatalogController {
             .await;
         Ok((
             ReleaseContext {
+                database_id,
                 streaming_job_ids: streaming_jobs,
                 state_table_ids,
                 source_ids,
@@ -430,6 +439,18 @@ impl CatalogController {
         ))
     }
 
+    pub async fn get_object_database_id(&self, object_id: ObjectId) -> MetaResult<DatabaseId> {
+        let inner = self.inner.read().await;
+        let (database_id,): (Option<DatabaseId>,) = Object::find_by_id(object_id)
+            .select_only()
+            .select_column(object::Column::DatabaseId)
+            .into_tuple()
+            .one(&inner.db)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found("object", object_id))?;
+        Ok(database_id.ok_or_else(|| anyhow!("object has no database id: {object_id}"))?)
+    }
+
     pub async fn create_schema(&self, schema: PbSchema) -> MetaResult<NotificationVersion> {
         let inner = self.inner.write().await;
         let owner_id = schema.owner as _;
@@ -671,25 +692,42 @@ impl CatalogController {
         Ok(tables)
     }
 
-    pub async fn list_object_dependencies(&self) -> MetaResult<Vec<PbObjectDependencies>> {
-        let inner = self.inner.read().await;
+    pub async fn list_all_object_dependencies(&self) -> MetaResult<Vec<PbObjectDependencies>> {
+        self.list_object_dependencies(true).await
+    }
 
-        let dependencies: Vec<(ObjectId, ObjectId)> = ObjectDependency::find()
-            .select_only()
-            .columns([
-                object_dependency::Column::Oid,
-                object_dependency::Column::UsedBy,
-            ])
-            .join(
-                JoinType::InnerJoin,
-                object_dependency::Relation::Object1.def(),
-            )
-            .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
-            .filter(streaming_job::Column::JobStatus.eq(JobStatus::Created))
-            .into_tuple()
-            .all(&inner.db)
-            .await?;
+    pub async fn list_created_object_dependencies(&self) -> MetaResult<Vec<PbObjectDependencies>> {
+        self.list_object_dependencies(false).await
+    }
 
+    async fn list_object_dependencies(
+        &self,
+        include_creating: bool,
+    ) -> MetaResult<Vec<PbObjectDependencies>> {
+        let inner = self.inner.read().await;
+
+        let dependencies: Vec<(ObjectId, ObjectId)> = {
+            let filter = if include_creating {
+                Expr::value(true)
+            } else {
+                streaming_job::Column::JobStatus.eq(JobStatus::Created)
+            };
+            ObjectDependency::find()
+                .select_only()
+                .columns([
+                    object_dependency::Column::Oid,
+                    object_dependency::Column::UsedBy,
+                ])
+                .join(
+                    JoinType::InnerJoin,
+                    object_dependency::Relation::Object1.def(),
+                )
+                .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
+                .filter(filter)
+                .into_tuple()
+                .all(&inner.db)
+                .await?
+        };
         let mut obj_dependencies = dependencies
             .into_iter()
             .map(|(oid, used_by)| PbObjectDependencies {
@@ -720,20 +758,24 @@ impl CatalogController {
             }
         }));
 
-        let sink_dependencies: Vec<(SinkId, TableId)> = Sink::find()
-            .select_only()
-            .columns([sink::Column::SinkId, sink::Column::TargetTable])
-            .join(JoinType::InnerJoin, sink::Relation::Object.def())
-            .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
-            .filter(
+        let sink_dependencies: Vec<(SinkId, TableId)> = {
+            let filter = if include_creating {
+                sink::Column::TargetTable.is_not_null()
+            } else {
                 streaming_job::Column::JobStatus
                     .eq(JobStatus::Created)
-                    .and(sink::Column::TargetTable.is_not_null()),
-            )
-            .into_tuple()
-            .all(&inner.db)
-            .await?;
-
+                    .and(sink::Column::TargetTable.is_not_null())
+            };
+            Sink::find()
+                .select_only()
+                .columns([sink::Column::SinkId, sink::Column::TargetTable])
+                .join(JoinType::InnerJoin, sink::Relation::Object.def())
+                .join(JoinType::InnerJoin, object::Relation::StreamingJob.def())
+                .filter(filter)
+                .into_tuple()
+                .all(&inner.db)
+                .await?
+        };
         obj_dependencies.extend(sink_dependencies.into_iter().map(|(sink_id, table_id)| {
             PbObjectDependencies {
                 object_id: table_id as _,
@@ -741,22 +783,26 @@ impl CatalogController {
             }
         }));
 
-        let subscription_dependencies: Vec<(SubscriptionId, TableId)> = Subscription::find()
-            .select_only()
-            .columns([
-                subscription::Column::SubscriptionId,
-                subscription::Column::DependentTableId,
-            ])
-            .join(JoinType::InnerJoin, subscription::Relation::Object.def())
-            .filter(
+        let subscription_dependencies: Vec<(SubscriptionId, TableId)> = {
+            let filter = if include_creating {
+                subscription::Column::DependentTableId.is_not_null()
+            } else {
                 subscription::Column::SubscriptionState
                     .eq(Into::<i32>::into(SubscriptionState::Created))
-                    .and(subscription::Column::DependentTableId.is_not_null()),
-            )
-            .into_tuple()
-            .all(&inner.db)
-            .await?;
-
+                    .and(subscription::Column::DependentTableId.is_not_null())
+            };
+            Subscription::find()
+                .select_only()
+                .columns([
+                    subscription::Column::SubscriptionId,
+                    subscription::Column::DependentTableId,
+                ])
+                .join(JoinType::InnerJoin, subscription::Relation::Object.def())
+                .filter(filter)
+                .into_tuple()
+                .all(&inner.db)
+                .await?
+        };
         obj_dependencies.extend(subscription_dependencies.into_iter().map(
             |(subscription_id, table_id)| PbObjectDependencies {
                 object_id: subscription_id as _,
@@ -891,7 +937,7 @@ impl CatalogController {
 
         txn.commit().await?;
 
-        let relation_group = build_relation_group(
+        let relation_group = build_relation_group_for_delete(
             dirty_mview_objs
                 .into_iter()
                 .chain(dirty_mview_internal_table_objs.into_iter())
@@ -1189,6 +1235,9 @@ impl CatalogController {
         )
         .await?;
 
+        // handle secret ref
+        let secret_ids = get_referred_secret_ids_from_source(&pb_source)?;
+
         let source_obj = Self::create_object(
             &txn,
             ObjectType::Source,
@@ -1202,6 +1251,19 @@ impl CatalogController {
         let source: source::ActiveModel = pb_source.clone().into();
         Source::insert(source).exec(&txn).await?;
 
+        // add secret dependency
+        if !secret_ids.is_empty() {
+            ObjectDependency::insert_many(secret_ids.iter().map(|id| {
+                object_dependency::ActiveModel {
+                    oid: Set(*id as _),
+                    used_by: Set(source_id as _),
+                    ..Default::default()
+                }
+            }))
+            .exec(&txn)
+            .await?;
+        }
+
         txn.commit().await?;
 
         let version = self
@@ -2109,6 +2171,9 @@ impl CatalogController {
             .await?
             .ok_or_else(|| MetaError::catalog_id_not_found(object_type.as_str(), object_id))?;
         assert_eq!(obj.obj_type, object_type);
+        let database_id = obj
+            .database_id
+            .ok_or_else(|| anyhow!("dropped object should have database_id"))?;
 
         let mut to_drop_objects = match drop_mode {
             DropMode::Cascade => get_referring_objects_cascade(object_id, &txn).await?,
@@ -2273,6 +2338,18 @@ impl CatalogController {
             to_drop_objects.extend(to_drop_internal_table_objs);
         }
 
+        let to_drop_objects = to_drop_objects;
+
+        to_drop_objects.iter().for_each(|obj| {
+            if let Some(obj_database_id) = obj.database_id {
+                assert_eq!(
+                    database_id, obj_database_id,
+                    "dropped objects not in the same database: {:?}",
+                    obj
+                );
+            }
+        });
+
         let (source_fragments, removed_actors, removed_fragments) =
             resolve_source_register_info_for_jobs(&txn, to_drop_streaming_jobs.clone()).await?;
 
@@ -2305,7 +2382,7 @@ impl CatalogController {
 
         // notify about them.
         self.notify_users_update(user_infos).await;
-        let relation_group = build_relation_group(to_drop_objects);
+        let relation_group = build_relation_group_for_delete(to_drop_objects);
 
         let version = self
             .notify_frontend(NotificationOperation::Delete, relation_group)
@@ -2324,6 +2401,7 @@ impl CatalogController {
 
         Ok((
             ReleaseContext {
+                database_id,
                 streaming_job_ids: to_drop_streaming_jobs,
                 state_table_ids: to_drop_state_table_ids,
                 source_ids: to_drop_source_ids,
@@ -2426,139 +2504,104 @@ impl CatalogController {
         )
         .await?;
 
-        let mut to_update_relations = vec![];
         // rename relation.
-        macro_rules! rename_relation {
-            ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
-                let (mut relation, obj) = $entity::find_by_id($object_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .unwrap();
-                let obj = obj.unwrap();
-                let old_name = relation.name.clone();
-                relation.name = object_name.into();
-                if obj.obj_type != ObjectType::View {
-                    relation.definition = alter_relation_rename(&relation.definition, object_name);
-                }
-                let active_model = $table::ActiveModel {
-                    $identity: Set(relation.$identity),
-                    name: Set(object_name.into()),
-                    definition: Set(relation.definition.clone()),
-                    ..Default::default()
-                };
-                active_model.update(&txn).await?;
-                to_update_relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::$entity(ObjectModel(relation, obj).into())),
-                });
-                old_name
-            }};
-        }
+        let (mut to_update_relations, old_name) =
+            rename_relation(&txn, object_type, object_id, object_name).await?;
+        // rename referring relation name.
+        to_update_relations.extend(
+            rename_relation_refer(&txn, object_type, object_id, object_name, &old_name).await?,
+        );
 
-        // TODO: check is there any thing to change for shared source?
-        let old_name = match object_type {
-            ObjectType::Table => rename_relation!(Table, table, table_id, object_id),
-            ObjectType::Source => rename_relation!(Source, source, source_id, object_id),
-            ObjectType::Sink => rename_relation!(Sink, sink, sink_id, object_id),
-            ObjectType::Subscription => {
-                rename_relation!(Subscription, subscription, subscription_id, object_id)
-            }
-            ObjectType::View => rename_relation!(View, view, view_id, object_id),
-            ObjectType::Index => {
-                let (mut index, obj) = Index::find_by_id(object_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .unwrap();
-                index.name = object_name.into();
-                let index_table_id = index.index_table_id;
-                let old_name = rename_relation!(Table, table, table_id, index_table_id);
-
-                // the name of index and its associated table is the same.
-                let active_model = index::ActiveModel {
-                    index_id: Set(index.index_id),
-                    name: Set(object_name.into()),
-                    ..Default::default()
-                };
-                active_model.update(&txn).await?;
-                to_update_relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::Index(
-                        ObjectModel(index, obj.unwrap()).into(),
-                    )),
-                });
-                old_name
-            }
-            _ => unreachable!("only relation name can be altered."),
-        };
+        txn.commit().await?;
 
-        // rename referring relation name.
-        macro_rules! rename_relation_ref {
-            ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
-                let (mut relation, obj) = $entity::find_by_id($object_id)
-                    .find_also_related(Object)
-                    .one(&txn)
-                    .await?
-                    .unwrap();
-                relation.definition =
-                    alter_relation_rename_refs(&relation.definition, &old_name, object_name);
-                let active_model = $table::ActiveModel {
-                    $identity: Set(relation.$identity),
-                    definition: Set(relation.definition.clone()),
-                    ..Default::default()
-                };
-                active_model.update(&txn).await?;
-                to_update_relations.push(PbRelation {
-                    relation_info: Some(PbRelationInfo::$entity(
-                        ObjectModel(relation, obj.unwrap()).into(),
-                    )),
-                });
-            }};
-        }
-        let mut objs = get_referring_objects(object_id, &txn).await?;
-        if object_type == ObjectType::Table {
-            let incoming_sinks: I32Array = Table::find_by_id(object_id)
+        let version = self
+            .notify_frontend(
+                NotificationOperation::Update,
+                NotificationInfo::RelationGroup(PbRelationGroup {
+                    relations: to_update_relations,
+                }),
+            )
+            .await;
+
+        Ok(version)
+    }
+
+    pub async fn alter_swap_rename(
+        &self,
+        object_type: ObjectType,
+        object_id: ObjectId,
+        dst_object_id: ObjectId,
+    ) -> MetaResult<NotificationVersion> {
+        let inner = self.inner.write().await;
+        let txn = inner.db.begin().await?;
+        let dst_name: String = match object_type {
+            ObjectType::Table => Table::find_by_id(dst_object_id)
+                .select_only()
+                .column(table::Column::Name)
+                .into_tuple()
+                .one(&txn)
+                .await?
+                .ok_or_else(|| {
+                    MetaError::catalog_id_not_found(object_type.as_str(), dst_object_id)
+                })?,
+            ObjectType::Source => Source::find_by_id(dst_object_id)
+                .select_only()
+                .column(source::Column::Name)
+                .into_tuple()
+                .one(&txn)
+                .await?
+                .ok_or_else(|| {
+                    MetaError::catalog_id_not_found(object_type.as_str(), dst_object_id)
+                })?,
+            ObjectType::Sink => Sink::find_by_id(dst_object_id)
+                .select_only()
+                .column(sink::Column::Name)
+                .into_tuple()
+                .one(&txn)
+                .await?
+                .ok_or_else(|| {
+                    MetaError::catalog_id_not_found(object_type.as_str(), dst_object_id)
+                })?,
+            ObjectType::View => View::find_by_id(dst_object_id)
+                .select_only()
+                .column(view::Column::Name)
+                .into_tuple()
+                .one(&txn)
+                .await?
+                .ok_or_else(|| {
+                    MetaError::catalog_id_not_found(object_type.as_str(), dst_object_id)
+                })?,
+            ObjectType::Subscription => Subscription::find_by_id(dst_object_id)
                 .select_only()
-                .column(table::Column::IncomingSinks)
+                .column(subscription::Column::Name)
                 .into_tuple()
                 .one(&txn)
                 .await?
-                .ok_or_else(|| MetaError::catalog_id_not_found("table", object_id))?;
+                .ok_or_else(|| {
+                    MetaError::catalog_id_not_found(object_type.as_str(), dst_object_id)
+                })?,
+            _ => {
+                return Err(MetaError::permission_denied(format!(
+                    "swap rename not supported for object type: {:?}",
+                    object_type
+                )));
+            }
+        };
 
-            objs.extend(
-                incoming_sinks
-                    .into_inner()
-                    .into_iter()
-                    .map(|id| PartialObject {
-                        oid: id,
-                        obj_type: ObjectType::Sink,
-                        schema_id: None,
-                        database_id: None,
-                    }),
-            );
-        }
+        // rename relations.
+        let (mut to_update_relations, src_name) =
+            rename_relation(&txn, object_type, object_id, &dst_name).await?;
+        let (to_update_relations2, _) =
+            rename_relation(&txn, object_type, dst_object_id, &src_name).await?;
+        to_update_relations.extend(to_update_relations2);
+        // rename referring relation name.
+        to_update_relations.extend(
+            rename_relation_refer(&txn, object_type, object_id, &dst_name, &src_name).await?,
+        );
+        to_update_relations.extend(
+            rename_relation_refer(&txn, object_type, dst_object_id, &src_name, &dst_name).await?,
+        );
 
-        for obj in objs {
-            match obj.obj_type {
-                ObjectType::Table => rename_relation_ref!(Table, table, table_id, obj.oid),
-                ObjectType::Sink => rename_relation_ref!(Sink, sink, sink_id, obj.oid),
-                ObjectType::Subscription => {
-                    rename_relation_ref!(Subscription, subscription, subscription_id, obj.oid)
-                }
-                ObjectType::View => rename_relation_ref!(View, view, view_id, obj.oid),
-                ObjectType::Index => {
-                    let index_table_id: Option<TableId> = Index::find_by_id(obj.oid)
-                        .select_only()
-                        .column(index::Column::IndexTableId)
-                        .into_tuple()
-                        .one(&txn)
-                        .await?;
-                    rename_relation_ref!(Table, table, table_id, index_table_id.unwrap());
-                }
-                _ => {
-                    bail!("only table, sink, subscription, view and index depend on other objects.")
-                }
-            }
-        }
         txn.commit().await?;
 
         let version = self
@@ -3450,13 +3493,7 @@ async fn update_internal_tables(
     new_value: Value,
     relations_to_notify: &mut Vec<PbRelationInfo>,
 ) -> MetaResult<()> {
-    let internal_tables: Vec<TableId> = Table::find()
-        .select_only()
-        .column(table::Column::TableId)
-        .filter(table::Column::BelongsToJobId.eq(object_id))
-        .into_tuple()
-        .all(txn)
-        .await?;
+    let internal_tables = get_internal_tables_by_id(object_id, txn).await?;
 
     if !internal_tables.is_empty() {
         Object::update_many()
@@ -3482,6 +3519,8 @@ async fn update_internal_tables(
 #[cfg(test)]
 mod tests {
 
+    use risingwave_pb::catalog::StreamSourceInfo;
+
     use super::*;
 
     const TEST_DATABASE_ID: DatabaseId = 1;
@@ -3641,6 +3680,9 @@ mod tests {
   scan.startup.mode = 'earliest'
 ) FORMAT PLAIN ENCODE JSON"#
                 .to_string(),
+            info: Some(StreamSourceInfo {
+                ..Default::default()
+            }),
             ..Default::default()
         };
         mgr.create_source(pb_source).await?;
diff --git a/src/meta/src/controller/fragment.rs b/src/meta/src/controller/fragment.rs
index e118ad3c9683c..67ff347f6c103 100644
--- a/src/meta/src/controller/fragment.rs
+++ b/src/meta/src/controller/fragment.rs
@@ -19,16 +19,17 @@ use std::mem::swap;
 use anyhow::Context;
 use itertools::Itertools;
 use risingwave_common::bail;
-use risingwave_common::hash::{VnodeCountCompat, WorkerSlotId};
+use risingwave_common::hash::{VnodeCount, VnodeCountCompat, WorkerSlotId};
 use risingwave_common::util::stream_graph_visitor::visit_stream_node;
 use risingwave_common::util::worker_util::WorkerNodeId;
 use risingwave_meta_model::actor::ActorStatus;
 use risingwave_meta_model::fragment::DistributionType;
+use risingwave_meta_model::object::ObjectType;
 use risingwave_meta_model::prelude::{Actor, ActorDispatcher, Fragment, Sink, StreamingJob};
 use risingwave_meta_model::{
-    actor, actor_dispatcher, fragment, sink, streaming_job, ActorId, ActorUpstreamActors,
-    ConnectorSplits, ExprContext, FragmentId, I32Array, JobStatus, ObjectId, SinkId, SourceId,
-    StreamNode, StreamingParallelism, TableId, VnodeBitmap, WorkerId,
+    actor, actor_dispatcher, fragment, object, sink, source, streaming_job, table, ActorId,
+    ActorUpstreamActors, ConnectorSplits, DatabaseId, ExprContext, FragmentId, I32Array, JobStatus,
+    ObjectId, SinkId, SourceId, StreamNode, StreamingParallelism, TableId, VnodeBitmap, WorkerId,
 };
 use risingwave_meta_model_migration::{Alias, SelectStatement};
 use risingwave_pb::common::PbActorLocation;
@@ -51,9 +52,11 @@ use risingwave_pb::stream_plan::{
 use sea_orm::sea_query::Expr;
 use sea_orm::ActiveValue::Set;
 use sea_orm::{
-    ColumnTrait, DbErr, EntityTrait, JoinType, ModelTrait, PaginatorTrait, QueryFilter,
-    QuerySelect, RelationTrait, SelectGetableTuple, Selector, TransactionTrait, Value,
+    ColumnTrait, DbErr, EntityTrait, FromQueryResult, JoinType, ModelTrait, PaginatorTrait,
+    QueryFilter, QuerySelect, RelationTrait, SelectGetableTuple, Selector, TransactionTrait, Value,
 };
+use serde::{Deserialize, Serialize};
+use tracing::debug;
 
 use crate::controller::catalog::{CatalogController, CatalogControllerInner};
 use crate::controller::utils::{
@@ -61,7 +64,7 @@ use crate::controller::utils::{
     FragmentDesc, PartialActorLocation, PartialFragmentStateTables,
 };
 use crate::manager::LocalNotification;
-use crate::model::{TableFragments, TableParallelism};
+use crate::model::TableParallelism;
 use crate::stream::SplitAssignment;
 use crate::{MetaError, MetaResult};
 
@@ -69,17 +72,6 @@ use crate::{MetaError, MetaResult};
 pub struct InflightFragmentInfo {
     pub actors: HashMap<ActorId, WorkerNodeId>,
     pub state_table_ids: HashSet<TableId>,
-    pub is_injectable: bool,
-}
-
-pub struct ActorInfos {
-    pub fragment_infos: HashMap<FragmentId, InflightFragmentInfo>,
-}
-
-impl ActorInfos {
-    pub fn new(fragment_infos: HashMap<FragmentId, InflightFragmentInfo>) -> Self {
-        Self { fragment_infos }
-    }
 }
 
 #[derive(Clone, Debug)]
@@ -89,6 +81,17 @@ pub struct FragmentParallelismInfo {
     pub vnode_count: usize,
 }
 
+#[derive(Clone, Debug, FromQueryResult, Serialize, Deserialize)]
+#[serde(rename_all = "camelCase")] // for dashboard
+pub struct StreamingJobInfo {
+    pub job_id: ObjectId,
+    pub obj_type: ObjectType,
+    pub name: String,
+    pub job_status: JobStatus,
+    pub parallelism: StreamingParallelism,
+    pub max_parallelism: i32,
+}
+
 impl CatalogControllerInner {
     /// List all fragment vnode mapping info for all CREATED streaming jobs.
     pub async fn all_running_fragment_mappings(
@@ -496,7 +499,7 @@ impl CatalogController {
             actors: pb_actors,
             state_table_ids: pb_state_table_ids,
             upstream_fragment_ids: pb_upstream_fragment_ids,
-            maybe_vnode_count: Some(vnode_count as _),
+            maybe_vnode_count: VnodeCount::set(vnode_count).to_protobuf(),
         };
 
         Ok((pb_fragment, pb_actor_status, pb_actor_splits))
@@ -659,6 +662,24 @@ impl CatalogController {
         Ok(object_ids)
     }
 
+    pub async fn list_fragment_database_ids(
+        &self,
+        select_fragment_ids: Option<Vec<FragmentId>>,
+    ) -> MetaResult<Vec<(FragmentId, DatabaseId)>> {
+        let inner = self.inner.read().await;
+        let select = Fragment::find()
+            .select_only()
+            .column(fragment::Column::FragmentId)
+            .column(object::Column::DatabaseId)
+            .join(JoinType::InnerJoin, fragment::Relation::Object.def());
+        let select = if let Some(select_fragment_ids) = select_fragment_ids {
+            select.filter(fragment::Column::FragmentId.is_in(select_fragment_ids))
+        } else {
+            select
+        };
+        Ok(select.into_tuple().all(&inner.db).await?)
+    }
+
     pub async fn get_job_fragments_by_id(&self, job_id: ObjectId) -> MetaResult<PbTableFragments> {
         let inner = self.inner.read().await;
         let fragment_actors = Fragment::find()
@@ -700,19 +721,35 @@ impl CatalogController {
         )
     }
 
-    pub async fn list_streaming_job_states(
-        &self,
-    ) -> MetaResult<Vec<(ObjectId, JobStatus, StreamingParallelism, i32)>> {
+    pub async fn list_streaming_job_infos(&self) -> MetaResult<Vec<StreamingJobInfo>> {
         let inner = self.inner.read().await;
         let job_states = StreamingJob::find()
             .select_only()
+            .column(streaming_job::Column::JobId)
+            .join(JoinType::InnerJoin, streaming_job::Relation::Object.def())
+            .column(object::Column::ObjType)
+            .join(JoinType::LeftJoin, table::Relation::Object1.def().rev())
+            .join(JoinType::LeftJoin, source::Relation::Object.def().rev())
+            .join(JoinType::LeftJoin, sink::Relation::Object.def().rev())
+            .column_as(
+                Expr::if_null(
+                    Expr::col((table::Entity, table::Column::Name)),
+                    Expr::if_null(
+                        Expr::col((source::Entity, source::Column::Name)),
+                        Expr::if_null(
+                            Expr::col((sink::Entity, sink::Column::Name)),
+                            Expr::val("<unknown>"),
+                        ),
+                    ),
+                ),
+                "name",
+            )
             .columns([
-                streaming_job::Column::JobId,
                 streaming_job::Column::JobStatus,
                 streaming_job::Column::Parallelism,
                 streaming_job::Column::MaxParallelism,
             ])
-            .into_tuple()
+            .into_model()
             .all(&inner.db)
             .await?;
         Ok(job_states)
@@ -720,11 +757,14 @@ impl CatalogController {
 
     pub async fn get_max_parallelism_by_id(&self, job_id: ObjectId) -> MetaResult<usize> {
         let inner = self.inner.read().await;
-        let job = StreamingJob::find_by_id(job_id)
+        let max_parallelism: i32 = StreamingJob::find_by_id(job_id)
+            .select_only()
+            .column(streaming_job::Column::MaxParallelism)
+            .into_tuple()
             .one(&inner.db)
             .await?
             .ok_or_else(|| anyhow::anyhow!("job {} not found in database", job_id))?;
-        Ok(job.max_parallelism as usize)
+        Ok(max_parallelism as usize)
     }
 
     /// Get all actor ids in the target streaming jobs.
@@ -933,47 +973,71 @@ impl CatalogController {
 
     /// Used in [`crate::barrier::GlobalBarrierManager`], load all running actor that need to be sent or
     /// collected
-    pub async fn load_all_actors(&self) -> MetaResult<ActorInfos> {
+    pub async fn load_all_actors(
+        &self,
+    ) -> MetaResult<
+        HashMap<
+            risingwave_common::catalog::DatabaseId,
+            HashMap<
+                risingwave_common::catalog::TableId,
+                HashMap<crate::model::FragmentId, InflightFragmentInfo>,
+            >,
+        >,
+    > {
         let inner = self.inner.read().await;
-        let actor_info: Vec<(ActorId, WorkerId, FragmentId, i32, I32Array)> = Actor::find()
+        let actor_info: Vec<(
+            ActorId,
+            WorkerId,
+            FragmentId,
+            I32Array,
+            DatabaseId,
+            ObjectId,
+        )> = Actor::find()
             .select_only()
             .column(actor::Column::ActorId)
             .column(actor::Column::WorkerId)
             .column(fragment::Column::FragmentId)
-            .column(fragment::Column::FragmentTypeMask)
             .column(fragment::Column::StateTableIds)
+            .column(object::Column::DatabaseId)
+            .column(object::Column::Oid)
             .join(JoinType::InnerJoin, actor::Relation::Fragment.def())
+            .join(JoinType::InnerJoin, fragment::Relation::Object.def())
             .filter(actor::Column::Status.eq(ActorStatus::Running))
             .into_tuple()
             .all(&inner.db)
             .await?;
 
-        let mut fragment_infos = HashMap::new();
-
-        for (actor_id, worker_id, fragment_id, type_mask, state_table_ids) in actor_info {
+        let mut database_fragment_infos: HashMap<_, HashMap<_, HashMap<_, InflightFragmentInfo>>> =
+            HashMap::new();
+
+        for (actor_id, worker_id, fragment_id, state_table_ids, database_id, job_id) in actor_info {
+            let fragment_infos = database_fragment_infos
+                .entry(risingwave_common::catalog::DatabaseId::new(
+                    database_id as _,
+                ))
+                .or_default()
+                .entry(risingwave_common::catalog::TableId::new(job_id as _))
+                .or_default();
             let state_table_ids = state_table_ids.into_inner();
-            match fragment_infos.entry(fragment_id) {
+            match fragment_infos.entry(fragment_id as crate::model::FragmentId) {
                 Entry::Occupied(mut entry) => {
                     let info: &mut InflightFragmentInfo = entry.get_mut();
                     debug_assert_eq!(info.state_table_ids, state_table_ids.into_iter().collect());
                     assert!(info.actors.insert(actor_id as _, worker_id as _).is_none());
-                    assert_eq!(
-                        info.is_injectable,
-                        TableFragments::is_injectable(type_mask as _)
-                    );
                 }
                 Entry::Vacant(entry) => {
                     let state_table_ids = state_table_ids.into_iter().collect();
                     entry.insert(InflightFragmentInfo {
                         actors: HashMap::from_iter([(actor_id as _, worker_id as _)]),
                         state_table_ids,
-                        is_injectable: TableFragments::is_injectable(type_mask as _),
                     });
                 }
             }
         }
 
-        Ok(ActorInfos::new(fragment_infos))
+        debug!(?database_fragment_infos, "reload all actors");
+
+        Ok(database_fragment_infos)
     }
 
     pub async fn migrate_actors(
@@ -1497,7 +1561,7 @@ mod tests {
     use std::collections::{BTreeMap, HashMap};
 
     use itertools::Itertools;
-    use risingwave_common::hash::{ActorMapping, VirtualNode};
+    use risingwave_common::hash::{ActorMapping, VirtualNode, VnodeCount};
     use risingwave_common::util::iter_util::ZipEqDebug;
     use risingwave_common::util::stream_graph_visitor::visit_stream_node;
     use risingwave_meta_model::actor::ActorStatus;
@@ -1627,7 +1691,7 @@ mod tests {
                 .values()
                 .flat_map(|m| m.keys().map(|x| *x as _))
                 .collect(),
-            maybe_vnode_count: Some(VirtualNode::COUNT_FOR_TEST as _),
+            maybe_vnode_count: VnodeCount::for_test().to_protobuf(),
         };
 
         let pb_actor_status = (0..actor_count)
diff --git a/src/meta/src/controller/mod.rs b/src/meta/src/controller/mod.rs
index cd0bb8530acbb..c7cf45daad9e7 100644
--- a/src/meta/src/controller/mod.rs
+++ b/src/meta/src/controller/mod.rs
@@ -15,6 +15,7 @@
 use std::collections::BTreeMap;
 
 use anyhow::anyhow;
+use risingwave_common::hash::VnodeCount;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_meta_model::{
     connection, database, function, index, object, schema, secret, sink, source, subscription,
@@ -164,7 +165,7 @@ impl From<ObjectModel<table::Model>> for PbTable {
             created_at_cluster_version: value.1.created_at_cluster_version,
             retention_seconds: value.0.retention_seconds.map(|id| id as u32),
             cdc_table_id: value.0.cdc_table_id,
-            maybe_vnode_count: Some(value.0.vnode_count as _),
+            maybe_vnode_count: VnodeCount::set(value.0.vnode_count).to_protobuf(),
         }
     }
 }
diff --git a/src/meta/src/controller/rename.rs b/src/meta/src/controller/rename.rs
index 5f10a8946414d..12155951c7f83 100644
--- a/src/meta/src/controller/rename.rs
+++ b/src/meta/src/controller/rename.rs
@@ -155,8 +155,11 @@ impl QueryRewriter<'_> {
             for cte_table in &mut with.cte_tables {
                 match &mut cte_table.cte_inner {
                     risingwave_sqlparser::ast::CteInner::Query(query) => self.visit_query(query),
-                    risingwave_sqlparser::ast::CteInner::ChangeLog(from) => {
-                        *from = Ident::new_unchecked(self.to)
+                    risingwave_sqlparser::ast::CteInner::ChangeLog(name) => {
+                        let idx = name.0.len() - 1;
+                        if name.0[idx].real_value() == self.from {
+                            name.0[idx] = Ident::with_quote_unchecked('"', self.to);
+                        }
                     }
                 }
             }
diff --git a/src/meta/src/controller/streaming_job.rs b/src/meta/src/controller/streaming_job.rs
index e8adc309d10e2..8496904d44c0c 100644
--- a/src/meta/src/controller/streaming_job.rs
+++ b/src/meta/src/controller/streaming_job.rs
@@ -15,7 +15,9 @@
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::num::NonZeroUsize;
 
+use anyhow::anyhow;
 use itertools::Itertools;
+use risingwave_common::hash::VnodeCountCompat;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::stream_graph_visitor::visit_stream_node;
 use risingwave_common::{bail, current_cluster_version};
@@ -41,8 +43,7 @@ use risingwave_pb::meta::subscribe_response::{
     Info as NotificationInfo, Info, Operation as NotificationOperation, Operation,
 };
 use risingwave_pb::meta::{
-    PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup, PbTableFragments, Relation,
-    RelationGroup,
+    PbFragmentWorkerSlotMapping, PbRelation, PbRelationGroup, Relation, RelationGroup,
 };
 use risingwave_pb::source::{PbConnectorSplit, PbConnectorSplits};
 use risingwave_pb::stream_plan::stream_fragment_graph::Parallelism;
@@ -63,13 +64,13 @@ use crate::barrier::{ReplaceTablePlan, Reschedule};
 use crate::controller::catalog::CatalogController;
 use crate::controller::rename::ReplaceTableExprRewriter;
 use crate::controller::utils::{
-    build_relation_group, check_relation_name_duplicate, check_sink_into_table_cycle,
+    build_relation_group_for_delete, check_relation_name_duplicate, check_sink_into_table_cycle,
     ensure_object_id, ensure_user_id, get_fragment_actor_ids, get_fragment_mappings,
-    rebuild_fragment_mapping_from_actors, PartialObject,
+    get_internal_tables_by_id, rebuild_fragment_mapping_from_actors, PartialObject,
 };
 use crate::controller::ObjectModel;
 use crate::manager::{NotificationVersion, StreamingJob};
-use crate::model::{StreamContext, TableParallelism};
+use crate::model::{StreamContext, TableFragments, TableParallelism};
 use crate::stream::SplitAssignment;
 use crate::{MetaError, MetaResult};
 
@@ -99,6 +100,11 @@ impl CatalogController {
         Ok(obj.oid)
     }
 
+    /// Create catalogs for the streaming job, then notify frontend about them if the job is a
+    /// materialized view.
+    ///
+    /// Some of the fields in the given streaming job are placeholders, which will
+    /// be updated later in `prepare_streaming_job` and notify again in `finish_streaming_job`.
     pub async fn create_job_catalog(
         &self,
         streaming_job: &mut StreamingJob,
@@ -333,16 +339,23 @@ impl CatalogController {
         Ok(())
     }
 
+    /// Create catalogs for internal tables, then notify frontend about them if the job is a
+    /// materialized view.
+    ///
+    /// Some of the fields in the given "incomplete" internal tables are placeholders, which will
+    /// be updated later in `prepare_streaming_job` and notify again in `finish_streaming_job`.
+    ///
+    /// Returns a mapping from the temporary table id to the actual global table id.
     pub async fn create_internal_table_catalog(
         &self,
         job: &StreamingJob,
-        mut internal_tables: Vec<PbTable>,
+        mut incomplete_internal_tables: Vec<PbTable>,
     ) -> MetaResult<HashMap<u32, u32>> {
         let job_id = job.id() as ObjectId;
         let inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
         let mut table_id_map = HashMap::new();
-        for table in &mut internal_tables {
+        for table in &mut incomplete_internal_tables {
             let table_id = Self::create_object(
                 &txn,
                 ObjectType::Table,
@@ -354,10 +367,13 @@ impl CatalogController {
             .oid;
             table_id_map.insert(table.id, table_id as u32);
             table.id = table_id as _;
-            let mut table_model: table::ActiveModel = table.clone().into();
-            table_model.table_id = Set(table_id as _);
-            table_model.belongs_to_job_id = Set(Some(job_id));
-            table_model.fragment_id = NotSet;
+
+            let table_model = table::ActiveModel {
+                table_id: Set(table_id as _),
+                belongs_to_job_id: Set(Some(job_id)),
+                fragment_id: NotSet,
+                ..table.clone().into()
+            };
             Table::insert(table_model).exec(&txn).await?;
         }
         txn.commit().await?;
@@ -366,7 +382,7 @@ impl CatalogController {
             self.notify_frontend(
                 Operation::Add,
                 Info::RelationGroup(RelationGroup {
-                    relations: internal_tables
+                    relations: incomplete_internal_tables
                         .iter()
                         .map(|table| Relation {
                             relation_info: Some(RelationInfo::Table(table.clone())),
@@ -380,14 +396,18 @@ impl CatalogController {
         Ok(table_id_map)
     }
 
+    // TODO: In this function, we also update the `Table` model in the meta store.
+    // Given that we've ensured the tables inside `TableFragments` are complete, shall we consider
+    // making them the source of truth and performing a full replacement for those in the meta store?
     pub async fn prepare_streaming_job(
         &self,
-        table_fragment: PbTableFragments,
+        table_fragments: &TableFragments,
         streaming_job: &StreamingJob,
         for_replace: bool,
     ) -> MetaResult<()> {
         let fragment_actors =
-            Self::extract_fragment_and_actors_from_table_fragments(table_fragment)?;
+            Self::extract_fragment_and_actors_from_table_fragments(table_fragments.to_protobuf())?;
+        let all_tables = table_fragments.all_tables();
         let inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
 
@@ -399,7 +419,6 @@ impl CatalogController {
         for fragment in fragments {
             let fragment_id = fragment.fragment_id;
             let state_table_ids = fragment.state_table_ids.inner_ref().clone();
-            let vnode_count = fragment.vnode_count;
 
             let fragment = fragment.into_active_model();
             Fragment::insert(fragment).exec(&txn).await?;
@@ -408,10 +427,19 @@ impl CatalogController {
             // After table fragments are created, update them for all internal tables.
             if !for_replace {
                 for state_table_id in state_table_ids {
+                    // Table's vnode count is not always the fragment's vnode count, so we have to
+                    // look up the table from `TableFragments`.
+                    // See `ActorGraphBuilder::new`.
+                    let table = all_tables
+                        .get(&(state_table_id as u32))
+                        .unwrap_or_else(|| panic!("table {} not found", state_table_id));
+                    assert_eq!(table.fragment_id, fragment_id as u32);
+                    let vnode_count = table.vnode_count();
+
                     table::ActiveModel {
                         table_id: Set(state_table_id as _),
                         fragment_id: Set(Some(fragment_id)),
-                        vnode_count: Set(vnode_count),
+                        vnode_count: Set(vnode_count as _),
                         ..Default::default()
                     }
                     .update(&txn)
@@ -454,23 +482,27 @@ impl CatalogController {
     }
 
     /// `try_abort_creating_streaming_job` is used to abort the job that is under initial status or in `FOREGROUND` mode.
-    /// It returns true if the job is not found or aborted.
+    /// It returns (true, _) if the job is not found or aborted.
+    /// It returns (_, Some(`database_id`)) is the `database_id` of the `job_id` exists
     pub async fn try_abort_creating_streaming_job(
         &self,
         job_id: ObjectId,
         is_cancelled: bool,
-    ) -> MetaResult<bool> {
+    ) -> MetaResult<(bool, Option<DatabaseId>)> {
         let mut inner = self.inner.write().await;
         let txn = inner.db.begin().await?;
 
-        let cnt = Object::find_by_id(job_id).count(&txn).await?;
-        if cnt == 0 {
+        let obj = Object::find_by_id(job_id).one(&txn).await?;
+        let Some(obj) = obj else {
             tracing::warn!(
                 id = job_id,
                 "streaming job not found when aborting creating, might be cleaned by recovery"
             );
-            return Ok(true);
-        }
+            return Ok((true, None));
+        };
+        let database_id = obj
+            .database_id
+            .ok_or_else(|| anyhow!("obj has no database id: {:?}", obj))?;
 
         if !is_cancelled {
             let streaming_job = streaming_job::Entity::find_by_id(job_id).one(&txn).await?;
@@ -484,18 +516,12 @@ impl CatalogController {
                         id = job_id,
                         "streaming job is created in background and still in creating status"
                     );
-                    return Ok(false);
+                    return Ok((false, Some(database_id)));
                 }
             }
         }
 
-        let internal_table_ids: Vec<TableId> = Table::find()
-            .select_only()
-            .column(table::Column::TableId)
-            .filter(table::Column::BelongsToJobId.eq(job_id))
-            .into_tuple()
-            .all(&txn)
-            .await?;
+        let internal_table_ids = get_internal_tables_by_id(job_id, &txn).await?;
 
         // Get the notification info if the job is a materialized view.
         let table_obj = Table::find_by_id(job_id).one(&txn).await?;
@@ -565,10 +591,12 @@ impl CatalogController {
         txn.commit().await?;
 
         if !objs.is_empty() {
-            self.notify_frontend(Operation::Delete, build_relation_group(objs))
+            // We also have notified the frontend about these objects,
+            // so we need to notify the frontend to delete them here.
+            self.notify_frontend(Operation::Delete, build_relation_group_for_delete(objs))
                 .await;
         }
-        Ok(true)
+        Ok((true, Some(database_id)))
     }
 
     pub async fn post_collect_table_fragments(
@@ -683,12 +711,33 @@ impl CatalogController {
             ));
         }
 
+        // 3. check parallelism.
+        let original_max_parallelism: i32 = StreamingJobModel::find_by_id(id as ObjectId)
+            .select_only()
+            .column(streaming_job::Column::MaxParallelism)
+            .into_tuple()
+            .one(&txn)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found(ObjectType::Table.as_str(), id))?;
+
+        if original_max_parallelism != max_parallelism as i32 {
+            // We already override the max parallelism in `StreamFragmentGraph` before entering this function.
+            // This should not happen in normal cases.
+            bail!(
+                "cannot use a different max parallelism \
+                 when altering or creating/dropping a sink into an existing table, \
+                 original: {}, new: {}",
+                original_max_parallelism,
+                max_parallelism
+            );
+        }
+
         let parallelism = match specified_parallelism {
             None => StreamingParallelism::Adaptive,
             Some(n) => StreamingParallelism::Fixed(n.get() as _),
         };
 
-        // 3. create streaming object for new replace table.
+        // 4. create streaming object for new replace table.
         let obj_id = Self::create_streaming_job_obj(
             &txn,
             ObjectType::Table,
@@ -702,7 +751,7 @@ impl CatalogController {
         )
         .await?;
 
-        // 4. record dependency for new replace table.
+        // 5. record dependency for new replace table.
         ObjectDependency::insert(object_dependency::ActiveModel {
             oid: Set(id as _),
             used_by: Set(obj_id as _),
@@ -1007,25 +1056,24 @@ impl CatalogController {
         table.incoming_sinks = Set(incoming_sinks.into());
         let table = table.update(txn).await?;
 
-        // Fields including `fragment_id` and `vnode_count` were placeholder values before.
+        // Fields including `fragment_id` were placeholder values before.
         // After table fragments are created, update them for all internal tables.
-        let fragment_info: Vec<(FragmentId, I32Array, i32)> = Fragment::find()
+        let fragment_info: Vec<(FragmentId, I32Array)> = Fragment::find()
             .select_only()
             .columns([
                 fragment::Column::FragmentId,
                 fragment::Column::StateTableIds,
-                fragment::Column::VnodeCount,
             ])
             .filter(fragment::Column::JobId.eq(dummy_id))
             .into_tuple()
             .all(txn)
             .await?;
-        for (fragment_id, state_table_ids, vnode_count) in fragment_info {
+        for (fragment_id, state_table_ids) in fragment_info {
             for state_table_id in state_table_ids.into_inner() {
                 table::ActiveModel {
                     table_id: Set(state_table_id as _),
                     fragment_id: Set(Some(fragment_id)),
-                    vnode_count: Set(vnode_count),
+                    // No need to update `vnode_count` because it must remain the same.
                     ..Default::default()
                 }
                 .update(txn)
@@ -1445,7 +1493,6 @@ impl CatalogController {
                 removed_actors,
                 vnode_bitmap_updates,
                 actor_splits,
-                injectable: _,
                 newly_created_actors,
                 upstream_fragment_dispatcher_ids,
                 upstream_dispatcher_mapping,
diff --git a/src/meta/src/controller/user.rs b/src/meta/src/controller/user.rs
index 3a8d728c84fd9..5e319d93133bc 100644
--- a/src/meta/src/controller/user.rs
+++ b/src/meta/src/controller/user.rs
@@ -34,8 +34,9 @@ use sea_orm::{
 use crate::controller::catalog::CatalogController;
 use crate::controller::utils::{
     check_user_name_duplicate, ensure_privileges_not_referred, ensure_user_id,
-    extract_grant_obj_id, get_object_owner, get_referring_privileges_cascade, get_user_privilege,
-    list_user_info_by_ids, PartialUserPrivilege,
+    extract_grant_obj_id, get_index_state_tables_by_table_id, get_internal_tables_by_id,
+    get_object_owner, get_referring_privileges_cascade, get_user_privilege, list_user_info_by_ids,
+    PartialUserPrivilege,
 };
 use crate::manager::{NotificationVersion, IGNORED_NOTIFICATION_VERSION};
 use crate::{MetaError, MetaResult};
@@ -230,14 +231,31 @@ impl CatalogController {
         let mut privileges = vec![];
         for gp in new_grant_privileges {
             let id = extract_grant_obj_id(gp.get_object()?);
+            let internal_table_ids = get_internal_tables_by_id(id, &txn).await?;
+            let index_state_table_ids = get_index_state_tables_by_table_id(id, &txn).await?;
             for action_with_opt in &gp.action_with_opts {
+                let action = action_with_opt.get_action()?.into();
                 privileges.push(user_privilege::ActiveModel {
                     oid: Set(id),
                     granted_by: Set(grantor),
-                    action: Set(action_with_opt.get_action()?.into()),
+                    action: Set(action),
                     with_grant_option: Set(action_with_opt.with_grant_option),
                     ..Default::default()
                 });
+                if action == Action::Select {
+                    privileges.extend(
+                        internal_table_ids
+                            .iter()
+                            .chain(index_state_table_ids.iter())
+                            .map(|&tid| user_privilege::ActiveModel {
+                                oid: Set(tid),
+                                granted_by: Set(grantor),
+                                action: Set(Action::Select),
+                                with_grant_option: Set(action_with_opt.with_grant_option),
+                                ..Default::default()
+                            }),
+                    );
+                }
             }
         }
 
@@ -254,7 +272,7 @@ impl CatalogController {
                 let filter = user_privilege::Column::UserId
                     .eq(grantor)
                     .and(user_privilege::Column::Oid.eq(*privilege.oid.as_ref()))
-                    .and(user_privilege::Column::Action.eq(privilege.action.as_ref().clone()))
+                    .and(user_privilege::Column::Action.eq(*privilege.action.as_ref()))
                     .and(user_privilege::Column::WithGrantOption.eq(true));
                 let privilege_id: Option<PrivilegeId> = UserPrivilege::find()
                     .select_only()
@@ -351,12 +369,29 @@ impl CatalogController {
         let mut revoke_items = HashMap::new();
         for privilege in revoke_grant_privileges {
             let obj = extract_grant_obj_id(privilege.get_object()?);
+            let internal_table_ids = get_internal_tables_by_id(obj, &txn).await?;
+            let index_state_table_ids = get_index_state_tables_by_table_id(obj, &txn).await?;
+            let mut include_select = false;
             let actions = privilege
                 .action_with_opts
                 .iter()
-                .map(|ao| Action::from(ao.get_action().unwrap()))
+                .map(|ao| {
+                    let action = Action::from(ao.get_action().unwrap());
+                    if action == Action::Select {
+                        include_select = true;
+                    }
+                    action
+                })
                 .collect_vec();
             revoke_items.insert(obj, actions);
+            if include_select {
+                revoke_items.extend(
+                    internal_table_ids
+                        .iter()
+                        .chain(index_state_table_ids.iter())
+                        .map(|&tid| (tid, vec![Action::Select])),
+                );
+            }
         }
 
         let filter = if !revoke_user.is_super {
diff --git a/src/meta/src/controller/utils.rs b/src/meta/src/controller/utils.rs
index a9650388b8db8..84d3d4da6736c 100644
--- a/src/meta/src/controller/utils.rs
+++ b/src/meta/src/controller/utils.rs
@@ -17,17 +17,18 @@ use std::collections::{BTreeSet, HashMap, HashSet};
 use anyhow::{anyhow, Context};
 use itertools::Itertools;
 use risingwave_common::bitmap::Bitmap;
-use risingwave_common::hash;
 use risingwave_common::hash::{ActorMapping, WorkerSlotId, WorkerSlotMapping};
+use risingwave_common::{bail, hash};
 use risingwave_meta_model::actor::ActorStatus;
 use risingwave_meta_model::fragment::DistributionType;
 use risingwave_meta_model::object::ObjectType;
 use risingwave_meta_model::prelude::*;
+use risingwave_meta_model::table::TableType;
 use risingwave_meta_model::{
     actor, actor_dispatcher, connection, database, fragment, function, index, object,
     object_dependency, schema, secret, sink, source, subscription, table, user, user_privilege,
     view, ActorId, ConnectorSplits, DataTypeArray, DatabaseId, FragmentId, I32Array, ObjectId,
-    PrivilegeId, SchemaId, SourceId, StreamNode, UserId, VnodeBitmap, WorkerId,
+    PrivilegeId, SchemaId, SourceId, StreamNode, TableId, UserId, VnodeBitmap, WorkerId,
 };
 use risingwave_meta_model_migration::WithQuery;
 use risingwave_pb::catalog::{
@@ -49,12 +50,15 @@ use sea_orm::sea_query::{
     WithClause,
 };
 use sea_orm::{
-    ColumnTrait, ConnectionTrait, DerivePartialModel, EntityTrait, FromQueryResult, JoinType,
-    Order, PaginatorTrait, QueryFilter, QuerySelect, RelationTrait, Statement,
+    ColumnTrait, ConnectionTrait, DatabaseTransaction, DerivePartialModel, EntityTrait,
+    FromQueryResult, JoinType, Order, PaginatorTrait, QueryFilter, QuerySelect, RelationTrait, Set,
+    Statement,
 };
 use thiserror_ext::AsReport;
 
+use crate::controller::ObjectModel;
 use crate::{MetaError, MetaResult};
+
 /// This function will construct a query using recursive cte to find all objects[(id, `obj_type`)] that are used by the given object.
 ///
 /// # Examples
@@ -273,7 +277,7 @@ pub struct FragmentDesc {
     pub state_table_ids: I32Array,
     pub upstream_fragment_id: I32Array,
     pub parallelism: i64,
-    pub vnode_count: i64,
+    pub vnode_count: i32,
 }
 
 /// List all objects that are using the given one in a cascade way. It runs a recursive CTE to find all the dependencies.
@@ -732,6 +736,58 @@ pub fn construct_privilege_dependency_query(ids: Vec<PrivilegeId>) -> WithQuery
         .to_owned()
 }
 
+pub async fn get_internal_tables_by_id<C>(job_id: ObjectId, db: &C) -> MetaResult<Vec<TableId>>
+where
+    C: ConnectionTrait,
+{
+    let table_ids: Vec<TableId> = Table::find()
+        .select_only()
+        .column(table::Column::TableId)
+        .filter(
+            table::Column::TableType
+                .eq(TableType::Internal)
+                .and(table::Column::BelongsToJobId.eq(job_id)),
+        )
+        .into_tuple()
+        .all(db)
+        .await?;
+    Ok(table_ids)
+}
+
+pub async fn get_index_state_tables_by_table_id<C>(
+    table_id: TableId,
+    db: &C,
+) -> MetaResult<Vec<TableId>>
+where
+    C: ConnectionTrait,
+{
+    let mut index_table_ids: Vec<TableId> = Index::find()
+        .select_only()
+        .column(index::Column::IndexTableId)
+        .filter(index::Column::PrimaryTableId.eq(table_id))
+        .into_tuple()
+        .all(db)
+        .await?;
+
+    if !index_table_ids.is_empty() {
+        let internal_table_ids: Vec<TableId> = Table::find()
+            .select_only()
+            .column(table::Column::TableId)
+            .filter(
+                table::Column::TableType
+                    .eq(TableType::Internal)
+                    .and(table::Column::BelongsToJobId.is_in(index_table_ids.clone())),
+            )
+            .into_tuple()
+            .all(db)
+            .await?;
+
+        index_table_ids.extend(internal_table_ids.into_iter());
+    }
+
+    Ok(index_table_ids)
+}
+
 #[derive(Clone, DerivePartialModel, FromQueryResult)]
 #[sea_orm(entity = "UserPrivilege")]
 pub struct PartialUserPrivilege {
@@ -1082,7 +1138,13 @@ where
     ))
 }
 
-pub(crate) fn build_relation_group(relation_objects: Vec<PartialObject>) -> NotificationInfo {
+/// Build a relation group for notifying the deletion of the given objects.
+///
+/// Note that only id fields are filled in the relation info, as the arguments are partial objects.
+/// As a result, the returned notification info should only be used for deletion.
+pub(crate) fn build_relation_group_for_delete(
+    relation_objects: Vec<PartialObject>,
+) -> NotificationInfo {
     let mut relations = vec![];
     for obj in relation_objects {
         match obj.obj_type {
@@ -1171,6 +1233,170 @@ pub fn extract_external_table_name_from_definition(table_definition: &str) -> Op
     }
 }
 
+/// `rename_relation` renames the target relation and its definition,
+/// it commits the changes to the transaction and returns the updated relations and the old name.
+pub async fn rename_relation(
+    txn: &DatabaseTransaction,
+    object_type: ObjectType,
+    object_id: ObjectId,
+    object_name: &str,
+) -> MetaResult<(Vec<PbRelation>, String)> {
+    use sea_orm::ActiveModelTrait;
+
+    use crate::controller::rename::alter_relation_rename;
+
+    let mut to_update_relations = vec![];
+    // rename relation.
+    macro_rules! rename_relation {
+        ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
+            let (mut relation, obj) = $entity::find_by_id($object_id)
+                .find_also_related(Object)
+                .one(txn)
+                .await?
+                .unwrap();
+            let obj = obj.unwrap();
+            let old_name = relation.name.clone();
+            relation.name = object_name.into();
+            if obj.obj_type != ObjectType::View {
+                relation.definition = alter_relation_rename(&relation.definition, object_name);
+            }
+            let active_model = $table::ActiveModel {
+                $identity: Set(relation.$identity),
+                name: Set(object_name.into()),
+                definition: Set(relation.definition.clone()),
+                ..Default::default()
+            };
+            active_model.update(txn).await?;
+            to_update_relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::$entity(ObjectModel(relation, obj).into())),
+            });
+            old_name
+        }};
+    }
+    // TODO: check is there any thing to change for shared source?
+    let old_name = match object_type {
+        ObjectType::Table => rename_relation!(Table, table, table_id, object_id),
+        ObjectType::Source => rename_relation!(Source, source, source_id, object_id),
+        ObjectType::Sink => rename_relation!(Sink, sink, sink_id, object_id),
+        ObjectType::Subscription => {
+            rename_relation!(Subscription, subscription, subscription_id, object_id)
+        }
+        ObjectType::View => rename_relation!(View, view, view_id, object_id),
+        ObjectType::Index => {
+            let (mut index, obj) = Index::find_by_id(object_id)
+                .find_also_related(Object)
+                .one(txn)
+                .await?
+                .unwrap();
+            index.name = object_name.into();
+            let index_table_id = index.index_table_id;
+            let old_name = rename_relation!(Table, table, table_id, index_table_id);
+
+            // the name of index and its associated table is the same.
+            let active_model = index::ActiveModel {
+                index_id: sea_orm::ActiveValue::Set(index.index_id),
+                name: sea_orm::ActiveValue::Set(object_name.into()),
+                ..Default::default()
+            };
+            active_model.update(txn).await?;
+            to_update_relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::Index(
+                    ObjectModel(index, obj.unwrap()).into(),
+                )),
+            });
+            old_name
+        }
+        _ => unreachable!("only relation name can be altered."),
+    };
+
+    Ok((to_update_relations, old_name))
+}
+
+/// `rename_relation_refer` updates the definition of relations that refer to the target one,
+/// it commits the changes to the transaction and returns all the updated relations.
+pub async fn rename_relation_refer(
+    txn: &DatabaseTransaction,
+    object_type: ObjectType,
+    object_id: ObjectId,
+    object_name: &str,
+    old_name: &str,
+) -> MetaResult<Vec<PbRelation>> {
+    use sea_orm::ActiveModelTrait;
+
+    use crate::controller::rename::alter_relation_rename_refs;
+
+    let mut to_update_relations = vec![];
+    macro_rules! rename_relation_ref {
+        ($entity:ident, $table:ident, $identity:ident, $object_id:expr) => {{
+            let (mut relation, obj) = $entity::find_by_id($object_id)
+                .find_also_related(Object)
+                .one(txn)
+                .await?
+                .unwrap();
+            relation.definition =
+                alter_relation_rename_refs(&relation.definition, old_name, object_name);
+            let active_model = $table::ActiveModel {
+                $identity: Set(relation.$identity),
+                definition: Set(relation.definition.clone()),
+                ..Default::default()
+            };
+            active_model.update(txn).await?;
+            to_update_relations.push(PbRelation {
+                relation_info: Some(PbRelationInfo::$entity(
+                    ObjectModel(relation, obj.unwrap()).into(),
+                )),
+            });
+        }};
+    }
+    let mut objs = get_referring_objects(object_id, txn).await?;
+    if object_type == ObjectType::Table {
+        let incoming_sinks: I32Array = Table::find_by_id(object_id)
+            .select_only()
+            .column(table::Column::IncomingSinks)
+            .into_tuple()
+            .one(txn)
+            .await?
+            .ok_or_else(|| MetaError::catalog_id_not_found("table", object_id))?;
+
+        objs.extend(
+            incoming_sinks
+                .into_inner()
+                .into_iter()
+                .map(|id| PartialObject {
+                    oid: id,
+                    obj_type: ObjectType::Sink,
+                    schema_id: None,
+                    database_id: None,
+                }),
+        );
+    }
+
+    for obj in objs {
+        match obj.obj_type {
+            ObjectType::Table => rename_relation_ref!(Table, table, table_id, obj.oid),
+            ObjectType::Sink => rename_relation_ref!(Sink, sink, sink_id, obj.oid),
+            ObjectType::Subscription => {
+                rename_relation_ref!(Subscription, subscription, subscription_id, obj.oid)
+            }
+            ObjectType::View => rename_relation_ref!(View, view, view_id, obj.oid),
+            ObjectType::Index => {
+                let index_table_id: Option<TableId> = Index::find_by_id(obj.oid)
+                    .select_only()
+                    .column(index::Column::IndexTableId)
+                    .into_tuple()
+                    .one(txn)
+                    .await?;
+                rename_relation_ref!(Table, table, table_id, index_table_id.unwrap());
+            }
+            _ => {
+                bail!("only table, sink, subscription, view and index depend on other objects.")
+            }
+        }
+    }
+
+    Ok(to_update_relations)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/src/meta/src/dashboard/mod.rs b/src/meta/src/dashboard/mod.rs
index bec4b82f59c96..32a3388cbcc90 100644
--- a/src/meta/src/dashboard/mod.rs
+++ b/src/meta/src/dashboard/mod.rs
@@ -76,6 +76,7 @@ pub(super) mod handlers {
     use thiserror_ext::AsReport;
 
     use super::*;
+    use crate::controller::fragment::StreamingJobInfo;
 
     pub struct DashboardError(anyhow::Error);
     pub type Result<T> = std::result::Result<T, DashboardError>;
@@ -191,20 +192,17 @@ pub(super) mod handlers {
         Ok(Json(views))
     }
 
-    pub async fn list_fragments(
+    pub async fn list_streaming_jobs(
         Extension(srv): Extension<Service>,
-    ) -> Result<Json<Vec<PbTableFragments>>> {
-        let table_fragments = srv
+    ) -> Result<Json<Vec<StreamingJobInfo>>> {
+        let streaming_jobs = srv
             .metadata_manager
             .catalog_controller
-            .table_fragments()
+            .list_streaming_job_infos()
             .await
-            .map_err(err)?
-            .values()
-            .cloned()
-            .collect_vec();
+            .map_err(err)?;
 
-        Ok(Json(table_fragments))
+        Ok(Json(streaming_jobs))
     }
 
     /// In the ddl backpressure graph, we want to compute the backpressure between relations.
@@ -326,7 +324,7 @@ pub(super) mod handlers {
         let object_dependencies = srv
             .metadata_manager
             .catalog_controller
-            .list_object_dependencies()
+            .list_all_object_dependencies()
             .await
             .map_err(err)?;
 
@@ -508,7 +506,7 @@ impl DashboardService {
 
         let api_router = Router::new()
             .route("/clusters/:ty", get(list_clusters))
-            .route("/fragments2", get(list_fragments))
+            .route("/streaming_jobs", get(list_streaming_jobs))
             .route("/fragments/job_id/:job_id", get(list_fragments_by_job_id))
             .route("/relation_id_infos", get(get_relation_id_infos))
             .route(
diff --git a/src/meta/src/error.rs b/src/meta/src/error.rs
index 8bfe188d4a3fa..f1c3bb0ffdd8a 100644
--- a/src/meta/src/error.rs
+++ b/src/meta/src/error.rs
@@ -116,9 +116,6 @@ pub enum MetaErrorInner {
         SinkError,
     ),
 
-    #[error("AWS SDK error: {0}")]
-    Aws(#[source] BoxedError),
-
     #[error(transparent)]
     Internal(
         #[from]
@@ -132,6 +129,9 @@ pub enum MetaErrorInner {
 
     #[error("Integrity check failed")]
     IntegrityCheckFailed,
+
+    #[error("{0} has been deprecated, please use {1} instead.")]
+    Deprecated(String, String),
 }
 
 impl MetaError {
@@ -156,15 +156,6 @@ impl MetaError {
     }
 }
 
-impl<E> From<aws_sdk_ec2::error::SdkError<E>> for MetaError
-where
-    E: std::error::Error + Sync + Send + 'static,
-{
-    fn from(e: aws_sdk_ec2::error::SdkError<E>) -> Self {
-        MetaErrorInner::Aws(e.into()).into()
-    }
-}
-
 impl From<MetaError> for tonic::Status {
     fn from(err: MetaError) -> Self {
         use tonic::Code;
diff --git a/src/meta/src/hummock/manager/checkpoint.rs b/src/meta/src/hummock/manager/checkpoint.rs
index f678014d440c8..3949331098441 100644
--- a/src/meta/src/hummock/manager/checkpoint.rs
+++ b/src/meta/src/hummock/manager/checkpoint.rs
@@ -17,10 +17,8 @@ use std::ops::Bound::{Excluded, Included};
 use std::ops::{Deref, DerefMut};
 use std::sync::atomic::Ordering;
 
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    object_size_map, summarize_group_deltas,
-};
-use risingwave_hummock_sdk::version::HummockVersion;
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::object_size_map;
+use risingwave_hummock_sdk::version::{GroupDeltaCommon, HummockVersion};
 use risingwave_hummock_sdk::HummockVersionId;
 use risingwave_pb::hummock::hummock_version_checkpoint::{PbStaleObjects, StaleObjects};
 use risingwave_pb::hummock::{
@@ -140,7 +138,6 @@ impl HummockManager {
             drop(versioning_guard);
             let versioning = self.versioning.read().await;
             let context_info = self.context_info.read().await;
-            versioning.mark_objects_for_deletion(&context_info, &self.delete_object_tracker);
             let min_pinned_version_id = context_info.min_pinned_version_id();
             trigger_gc_stat(&self.metrics, &versioning.checkpoint, min_pinned_version_id);
             return Ok(0);
@@ -156,13 +153,27 @@ impl HummockManager {
             .hummock_version_deltas
             .range((Excluded(old_checkpoint_id), Included(new_checkpoint_id)))
         {
-            for (group_id, group_deltas) in &version_delta.group_deltas {
-                let summary = summarize_group_deltas(group_deltas, *group_id);
+            for group_deltas in version_delta.group_deltas.values() {
                 object_sizes.extend(
-                    summary
-                        .insert_table_infos
+                    group_deltas
+                        .group_deltas
                         .iter()
-                        .map(|t| (t.object_id, t.file_size))
+                        .flat_map(|delta| {
+                            match delta {
+                                GroupDeltaCommon::IntraLevel(level_delta) => {
+                                    Some(level_delta.inserted_table_infos.iter())
+                                }
+                                GroupDeltaCommon::NewL0SubLevel(inserted_table_infos) => {
+                                    Some(inserted_table_infos.iter())
+                                }
+                                GroupDeltaCommon::GroupConstruct(_)
+                                | GroupDeltaCommon::GroupDestroy(_)
+                                | GroupDeltaCommon::GroupMerge(_) => None,
+                            }
+                            .into_iter()
+                            .flatten()
+                            .map(|t| (t.object_id, t.file_size))
+                        })
                         .chain(
                             version_delta
                                 .change_log_delta
@@ -209,12 +220,9 @@ impl HummockManager {
                     .collect(),
             });
         }
-        // Whenever data archive or time travel is enabled, we can directly discard reference to stale objects that will no longer be used.
-        if self.env.opts.enable_hummock_data_archive || self.time_travel_enabled().await {
-            let context_info = self.context_info.read().await;
-            let min_pinned_version_id = context_info.min_pinned_version_id();
-            stale_objects.retain(|version_id, _| *version_id >= min_pinned_version_id);
-        }
+        // We can directly discard reference to stale objects that will no longer be used.
+        let min_pinned_version_id = self.context_info.read().await.min_pinned_version_id();
+        stale_objects.retain(|version_id, _| *version_id >= min_pinned_version_id);
         let new_checkpoint = HummockVersionCheckpoint {
             version: current_version.clone(),
             stale_objects,
@@ -234,15 +242,9 @@ impl HummockManager {
         // 3. hold write lock and update in memory state
         let mut versioning_guard = self.versioning.write().await;
         let versioning = versioning_guard.deref_mut();
-        let context_info = self.context_info.read().await;
         assert!(new_checkpoint.version.id > versioning.checkpoint.version.id);
         versioning.checkpoint = new_checkpoint;
-        // Not delete stale objects when archive or time travel is enabled
-        if !self.env.opts.enable_hummock_data_archive && !self.time_travel_enabled().await {
-            versioning.mark_objects_for_deletion(&context_info, &self.delete_object_tracker);
-        }
-
-        let min_pinned_version_id = context_info.min_pinned_version_id();
+        let min_pinned_version_id = self.context_info.read().await.min_pinned_version_id();
         trigger_gc_stat(&self.metrics, &versioning.checkpoint, min_pinned_version_id);
         trigger_split_stat(&self.metrics, &versioning.current_version);
         drop(versioning_guard);
diff --git a/src/meta/src/hummock/manager/commit_epoch.rs b/src/meta/src/hummock/manager/commit_epoch.rs
index da97a29c06f8c..c51c77a5d36a0 100644
--- a/src/meta/src/hummock/manager/commit_epoch.rs
+++ b/src/meta/src/hummock/manager/commit_epoch.rs
@@ -16,9 +16,9 @@ use std::collections::{BTreeMap, HashMap, HashSet};
 use std::sync::Arc;
 
 use risingwave_common::catalog::TableId;
+use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_hummock_sdk::change_log::ChangeLogDelta;
 use risingwave_hummock_sdk::compaction_group::group_split::split_sst_with_table_ids;
-use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::{
     add_prost_table_stats_map, purge_prost_table_stats, to_prost_table_stats_map, PbTableStatsMap,
@@ -43,19 +43,13 @@ use crate::hummock::metrics_utils::{
 };
 use crate::hummock::model::CompactionGroup;
 use crate::hummock::sequence::{next_compaction_group_id, next_sstable_object_id};
+use crate::hummock::time_travel::should_mark_next_time_travel_version_snapshot;
 use crate::hummock::{
-    commit_multi_var, commit_multi_var_with_provided_txn, start_measure_real_process_timer,
-    HummockManager,
+    commit_multi_var_with_provided_txn, start_measure_real_process_timer, HummockManager,
 };
 
-pub enum NewTableFragmentInfo {
-    Normal {
-        mv_table_id: Option<TableId>,
-        internal_table_ids: Vec<TableId>,
-    },
-    NewCompactionGroup {
-        table_ids: HashSet<TableId>,
-    },
+pub struct NewTableFragmentInfo {
+    pub table_ids: HashSet<TableId>,
 }
 
 #[derive(Default)]
@@ -108,13 +102,6 @@ impl HummockManager {
             );
         }
 
-        let previous_time_travel_toggle_check = versioning.time_travel_toggle_check;
-        versioning.time_travel_toggle_check = self.time_travel_enabled().await;
-        if !previous_time_travel_toggle_check && versioning.time_travel_toggle_check {
-            // Take a snapshot for the first commit epoch after enabling time travel.
-            versioning.mark_next_time_travel_version_snapshot();
-        }
-
         let mut version = HummockVersionTransaction::new(
             &mut versioning.current_version,
             &mut versioning.hummock_version_deltas,
@@ -130,73 +117,48 @@ impl HummockManager {
         let mut compaction_group_config: Option<Arc<CompactionConfig>> = None;
 
         // Add new table
-        for new_table_fragment_info in new_table_fragment_infos {
-            match new_table_fragment_info {
-                NewTableFragmentInfo::Normal {
-                    mv_table_id,
-                    internal_table_ids,
-                } => {
-                    on_handle_add_new_table(
-                        state_table_info,
-                        &internal_table_ids,
-                        StaticCompactionGroupId::StateDefault as u64,
-                        &mut table_compaction_group_mapping,
-                        &mut new_table_ids,
-                    )?;
-
-                    on_handle_add_new_table(
-                        state_table_info,
-                        &mv_table_id,
-                        StaticCompactionGroupId::MaterializedView as u64,
-                        &mut table_compaction_group_mapping,
-                        &mut new_table_ids,
-                    )?;
-                }
-                NewTableFragmentInfo::NewCompactionGroup { table_ids } => {
-                    let (compaction_group_manager, compaction_group_config) =
-                        if let Some(compaction_group_manager) = &mut compaction_group_manager_txn {
-                            (
-                                compaction_group_manager,
-                                (*compaction_group_config
-                                    .as_ref()
-                                    .expect("must be set with compaction_group_manager_txn"))
-                                .clone(),
-                            )
-                        } else {
-                            let compaction_group_manager_guard =
-                                self.compaction_group_manager.write().await;
-                            let new_compaction_group_config =
-                                compaction_group_manager_guard.default_compaction_config();
-                            compaction_group_config = Some(new_compaction_group_config.clone());
-                            (
-                                compaction_group_manager_txn.insert(
-                                    CompactionGroupManager::start_owned_compaction_groups_txn(
-                                        compaction_group_manager_guard,
-                                    ),
-                                ),
-                                new_compaction_group_config,
-                            )
-                        };
-                    let new_compaction_group_id = next_compaction_group_id(&self.env).await?;
-                    new_compaction_groups
-                        .insert(new_compaction_group_id, compaction_group_config.clone());
-                    compaction_group_manager.insert(
-                        new_compaction_group_id,
-                        CompactionGroup {
-                            group_id: new_compaction_group_id,
-                            compaction_config: compaction_group_config,
-                        },
-                    );
-
-                    on_handle_add_new_table(
-                        state_table_info,
-                        &table_ids,
-                        new_compaction_group_id,
-                        &mut table_compaction_group_mapping,
-                        &mut new_table_ids,
-                    )?;
-                }
-            }
+        for NewTableFragmentInfo { table_ids } in new_table_fragment_infos {
+            let (compaction_group_manager, compaction_group_config) =
+                if let Some(compaction_group_manager) = &mut compaction_group_manager_txn {
+                    (
+                        compaction_group_manager,
+                        (*compaction_group_config
+                            .as_ref()
+                            .expect("must be set with compaction_group_manager_txn"))
+                        .clone(),
+                    )
+                } else {
+                    let compaction_group_manager_guard =
+                        self.compaction_group_manager.write().await;
+                    let new_compaction_group_config =
+                        compaction_group_manager_guard.default_compaction_config();
+                    compaction_group_config = Some(new_compaction_group_config.clone());
+                    (
+                        compaction_group_manager_txn.insert(
+                            CompactionGroupManager::start_owned_compaction_groups_txn(
+                                compaction_group_manager_guard,
+                            ),
+                        ),
+                        new_compaction_group_config,
+                    )
+                };
+            let new_compaction_group_id = next_compaction_group_id(&self.env).await?;
+            new_compaction_groups.insert(new_compaction_group_id, compaction_group_config.clone());
+            compaction_group_manager.insert(
+                new_compaction_group_id,
+                CompactionGroup {
+                    group_id: new_compaction_group_id,
+                    compaction_config: compaction_group_config,
+                },
+            );
+
+            on_handle_add_new_table(
+                state_table_info,
+                &table_ids,
+                new_compaction_group_id,
+                &mut table_compaction_group_mapping,
+                &mut new_table_ids,
+            )?;
         }
 
         let commit_sstables = self
@@ -213,6 +175,10 @@ impl HummockManager {
             new_table_watermarks,
             change_log_delta,
         );
+        if should_mark_next_time_travel_version_snapshot(&time_travel_delta) {
+            // Unable to invoke mark_next_time_travel_version_snapshot because versioning is already mutable borrowed.
+            versioning.time_travel_snapshot_interval_counter = u64::MAX;
+        }
 
         // Apply stats changes.
         let mut version_stats = HummockVersionStatsTransaction::new(
@@ -246,59 +212,50 @@ impl HummockManager {
             );
             table_metrics.inc_write_throughput(stats_value as u64);
         }
-        if versioning.time_travel_toggle_check {
-            let mut time_travel_version = None;
-            if versioning.time_travel_snapshot_interval_counter
-                >= self.env.opts.hummock_time_travel_snapshot_interval
-            {
-                versioning.time_travel_snapshot_interval_counter = 0;
-                time_travel_version = Some(version.latest_version());
-            } else {
-                versioning.time_travel_snapshot_interval_counter = versioning
-                    .time_travel_snapshot_interval_counter
-                    .saturating_add(1);
-            }
-            let group_parents = version
-                .latest_version()
-                .levels
-                .values()
-                .map(|g| (g.group_id, g.parent_group_id))
-                .collect();
-            let time_travel_tables_to_commit =
-                table_compaction_group_mapping
-                    .iter()
-                    .filter_map(|(table_id, cg_id)| {
-                        tables_to_commit
-                            .get(table_id)
-                            .map(|committed_epoch| (table_id, cg_id, *committed_epoch))
-                    });
-            let mut txn = self.env.meta_store_ref().conn.begin().await?;
-            let version_snapshot_sst_ids = self
-                .write_time_travel_metadata(
-                    &txn,
-                    time_travel_version,
-                    time_travel_delta,
-                    &group_parents,
-                    &versioning.last_time_travel_snapshot_sst_ids,
-                    time_travel_tables_to_commit,
-                )
-                .await?;
-            commit_multi_var_with_provided_txn!(
-                txn,
-                version,
-                version_stats,
-                compaction_group_manager_txn
-            )?;
-            if let Some(version_snapshot_sst_ids) = version_snapshot_sst_ids {
-                versioning.last_time_travel_snapshot_sst_ids = version_snapshot_sst_ids;
-            }
+        let mut time_travel_version = None;
+        if versioning.time_travel_snapshot_interval_counter
+            >= self.env.opts.hummock_time_travel_snapshot_interval
+        {
+            versioning.time_travel_snapshot_interval_counter = 0;
+            time_travel_version = Some(version.latest_version());
         } else {
-            commit_multi_var!(
-                self.meta_store_ref(),
-                version,
-                version_stats,
-                compaction_group_manager_txn
-            )?;
+            versioning.time_travel_snapshot_interval_counter = versioning
+                .time_travel_snapshot_interval_counter
+                .saturating_add(1);
+        }
+        let group_parents = version
+            .latest_version()
+            .levels
+            .values()
+            .map(|g| (g.group_id, g.parent_group_id))
+            .collect();
+        let time_travel_tables_to_commit =
+            table_compaction_group_mapping
+                .iter()
+                .filter_map(|(table_id, cg_id)| {
+                    tables_to_commit
+                        .get(table_id)
+                        .map(|committed_epoch| (table_id, cg_id, *committed_epoch))
+                });
+        let mut txn = self.env.meta_store_ref().conn.begin().await?;
+        let version_snapshot_sst_ids = self
+            .write_time_travel_metadata(
+                &txn,
+                time_travel_version,
+                time_travel_delta,
+                &group_parents,
+                &versioning.last_time_travel_snapshot_sst_ids,
+                time_travel_tables_to_commit,
+            )
+            .await?;
+        commit_multi_var_with_provided_txn!(
+            txn,
+            version,
+            version_stats,
+            compaction_group_manager_txn
+        )?;
+        if let Some(version_snapshot_sst_ids) = version_snapshot_sst_ids {
+            versioning.last_time_travel_snapshot_sst_ids = version_snapshot_sst_ids;
         }
 
         for compaction_group_id in &modified_compaction_groups {
@@ -319,7 +276,8 @@ impl HummockManager {
                 self.try_send_compaction_request(*id, compact_task::TaskType::Dynamic);
             }
             if !table_stats_change.is_empty() {
-                self.collect_table_write_throughput(table_stats_change);
+                self.collect_table_write_throughput(table_stats_change)
+                    .await;
             }
         }
         if !modified_compaction_groups.is_empty() {
@@ -333,13 +291,30 @@ impl HummockManager {
         Ok(())
     }
 
-    fn collect_table_write_throughput(&self, table_stats: PbTableStatsMap) {
+    async fn collect_table_write_throughput(&self, table_stats: PbTableStatsMap) {
+        let params = self.env.system_params_reader().await;
+        let barrier_interval_ms = params.barrier_interval_ms() as u64;
+        let checkpoint_secs = {
+            std::cmp::max(
+                1,
+                params.checkpoint_frequency() * barrier_interval_ms / 1000,
+            )
+        };
+
         let mut table_infos = self.history_table_throughput.write();
+        let max_table_stat_throuput_window_seconds = std::cmp::max(
+            self.env.opts.table_stat_throuput_window_seconds_for_split,
+            self.env.opts.table_stat_throuput_window_seconds_for_merge,
+        );
+
+        let max_sample_size = (max_table_stat_throuput_window_seconds as f64
+            / checkpoint_secs as f64)
+            .ceil() as usize;
         for (table_id, stat) in table_stats {
             let throughput = (stat.total_value_size + stat.total_key_size) as u64;
             let entry = table_infos.entry(table_id).or_default();
             entry.push_back(throughput);
-            if entry.len() > self.env.opts.table_info_statistic_history_times {
+            if entry.len() > max_sample_size {
                 entry.pop_front();
             }
         }
diff --git a/src/meta/src/hummock/manager/compaction/compaction_group_manager.rs b/src/meta/src/hummock/manager/compaction/compaction_group_manager.rs
index 77822a0d6f3cc..02b63ab47de62 100644
--- a/src/meta/src/hummock/manager/compaction/compaction_group_manager.rs
+++ b/src/meta/src/hummock/manager/compaction/compaction_group_manager.rs
@@ -19,9 +19,7 @@ use std::sync::Arc;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::epoch::INVALID_EPOCH;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    get_compaction_group_ids, TableGroupInfo,
-};
+use risingwave_hummock_sdk::compaction_group::hummock_version_ext::get_compaction_group_ids;
 use risingwave_hummock_sdk::compaction_group::{StateTableId, StaticCompactionGroupId};
 use risingwave_hummock_sdk::version::GroupDelta;
 use risingwave_hummock_sdk::CompactionGroupId;
@@ -34,6 +32,7 @@ use risingwave_pb::hummock::{
 use sea_orm::EntityTrait;
 use tokio::sync::OnceCell;
 
+use super::CompactionGroupStatistic;
 use crate::hummock::compaction::compaction_config::{
     validate_compaction_config, CompactionConfigBuilder,
 };
@@ -418,13 +417,14 @@ impl HummockManager {
         results
     }
 
-    pub async fn calculate_compaction_group_statistic(&self) -> Vec<TableGroupInfo> {
+    pub async fn calculate_compaction_group_statistic(&self) -> Vec<CompactionGroupStatistic> {
         let mut infos = vec![];
         {
             let versioning_guard = self.versioning.read().await;
+            let manager = self.compaction_group_manager.read().await;
             let version = &versioning_guard.current_version;
             for group_id in version.levels.keys() {
-                let mut group_info = TableGroupInfo {
+                let mut group_info = CompactionGroupStatistic {
                     group_id: *group_id,
                     ..Default::default()
                 };
@@ -443,16 +443,12 @@ impl HummockManager {
                     group_info
                         .table_statistic
                         .insert(table_id.table_id, table_size);
+                    group_info.compaction_group_config =
+                        manager.try_get_compaction_group_config(*group_id).unwrap();
                 }
                 infos.push(group_info);
             }
         };
-        let manager = self.compaction_group_manager.read().await;
-        for info in &mut infos {
-            if let Some(group) = manager.compaction_groups.get(&info.group_id) {
-                info.split_by_table = group.compaction_config.split_by_state_table;
-            }
-        }
         infos
     }
 
@@ -493,6 +489,7 @@ impl CompactionGroupManager {
         CompactionGroupTransaction::new(&mut self.compaction_groups)
     }
 
+    #[expect(clippy::type_complexity)]
     pub fn start_owned_compaction_groups_txn<P: DerefMut<Target = Self>>(
         inner: P,
     ) -> BTreeMapTransactionInner<
@@ -595,7 +592,7 @@ fn update_compaction_config(target: &mut CompactionConfig, items: &[MutableConfi
     }
 }
 
-impl<'a> CompactionGroupTransaction<'a> {
+impl CompactionGroupTransaction<'_> {
     /// Inserts compaction group configs if they do not exist.
     pub fn try_create_compaction_groups(
         &mut self,
diff --git a/src/meta/src/hummock/manager/compaction/compaction_group_schedule.rs b/src/meta/src/hummock/manager/compaction/compaction_group_schedule.rs
index 975d764c9aed8..578a438225d07 100644
--- a/src/meta/src/hummock/manager/compaction/compaction_group_schedule.rs
+++ b/src/meta/src/hummock/manager/compaction/compaction_group_schedule.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
 use std::ops::DerefMut;
 use std::sync::Arc;
 
@@ -21,9 +21,6 @@ use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::compact_task::ReportTask;
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::{
-    HummockLevelsExt, TableGroupInfo,
-};
 use risingwave_hummock_sdk::compaction_group::{
     group_split, StateTableId, StaticCompactionGroupId,
 };
@@ -36,6 +33,7 @@ use risingwave_pb::hummock::{
 };
 use thiserror_ext::AsReport;
 
+use super::CompactionGroupStatistic;
 use crate::hummock::error::{Error, Result};
 use crate::hummock::manager::transaction::HummockVersionTransaction;
 use crate::hummock::manager::{commit_multi_var, HummockManager};
@@ -47,6 +45,26 @@ impl HummockManager {
         &self,
         group_1: CompactionGroupId,
         group_2: CompactionGroupId,
+    ) -> Result<()> {
+        self.merge_compaction_group_impl(group_1, group_2, None)
+            .await
+    }
+
+    pub async fn merge_compaction_group_for_test(
+        &self,
+        group_1: CompactionGroupId,
+        group_2: CompactionGroupId,
+        created_tables: HashSet<u32>,
+    ) -> Result<()> {
+        self.merge_compaction_group_impl(group_1, group_2, Some(created_tables))
+            .await
+    }
+
+    pub async fn merge_compaction_group_impl(
+        &self,
+        group_1: CompactionGroupId,
+        group_2: CompactionGroupId,
+        created_tables: Option<HashSet<u32>>,
     ) -> Result<()> {
         let compaction_guard = self.compaction.write().await;
         let mut versioning_guard = self.versioning.write().await;
@@ -78,6 +96,42 @@ impl HummockManager {
         assert!(member_table_ids_1.is_sorted());
         assert!(member_table_ids_2.is_sorted());
 
+        let created_tables = if let Some(created_tables) = created_tables {
+            // if the created_tables is provided, use it directly, most for test
+            assert!(cfg!(debug_assertions));
+            created_tables
+        } else {
+            match self.metadata_manager.get_created_table_ids().await {
+                Ok(created_tables) => HashSet::from_iter(created_tables),
+                Err(err) => {
+                    tracing::warn!(error = %err.as_report(), "failed to fetch created table ids");
+                    return Err(Error::CompactionGroup(format!(
+                        "merge group_1 {} group_2 {} failed to fetch created table ids",
+                        group_1, group_2
+                    )));
+                }
+            }
+        };
+
+        fn contains_creating_table(
+            table_ids: &Vec<TableId>,
+            created_tables: &HashSet<u32>,
+        ) -> bool {
+            table_ids
+                .iter()
+                .any(|table_id| !created_tables.contains(&table_id.table_id()))
+        }
+
+        // do not merge the compaction group which is creating
+        if contains_creating_table(&member_table_ids_1, &created_tables)
+            || contains_creating_table(&member_table_ids_2, &created_tables)
+        {
+            return Err(Error::CompactionGroup(format!(
+                "Not Merge creating group {} next_group {} member_table_ids_1 {:?} member_table_ids_2 {:?}",
+                group_1, group_2, member_table_ids_1, member_table_ids_2
+            )));
+        }
+
         // Make sure `member_table_ids_1` is smaller than `member_table_ids_2`
         let (left_group_id, right_group_id) =
             if member_table_ids_1.first().unwrap() < member_table_ids_2.first().unwrap() {
@@ -265,123 +319,6 @@ impl HummockManager {
 
         Ok(())
     }
-
-    pub async fn try_split_compaction_group(
-        &self,
-        table_write_throughput: &HashMap<u32, VecDeque<u64>>,
-        checkpoint_secs: u64,
-        group: &TableGroupInfo,
-        created_tables: &HashSet<u32>,
-    ) {
-        let compaction_group_config = {
-            let compaction_group_manager = self.compaction_group_manager.read().await;
-            compaction_group_manager
-                .try_get_compaction_group_config(group.group_id)
-                .unwrap()
-        };
-
-        if compaction_group_config
-            .compaction_config
-            .disable_auto_group_scheduling
-            .unwrap_or(false)
-        {
-            return;
-        }
-
-        // split high throughput table to dedicated compaction group
-        for (table_id, table_size) in &group.table_statistic {
-            self.try_move_table_to_dedicated_cg(
-                table_write_throughput,
-                table_id,
-                table_size,
-                !created_tables.contains(table_id),
-                checkpoint_secs,
-                group.group_id,
-                group.group_size,
-            )
-            .await;
-        }
-    }
-
-    pub async fn try_move_table_to_dedicated_cg(
-        &self,
-        table_write_throughput: &HashMap<u32, VecDeque<u64>>,
-        table_id: &u32,
-        table_size: &u64,
-        is_creating_table: bool,
-        checkpoint_secs: u64,
-        parent_group_id: u64,
-        group_size: u64,
-    ) {
-        let default_group_id: CompactionGroupId = StaticCompactionGroupId::StateDefault.into();
-        let mv_group_id: CompactionGroupId = StaticCompactionGroupId::MaterializedView.into();
-        let partition_vnode_count = self.env.opts.partition_vnode_count;
-        let window_size =
-            self.env.opts.table_info_statistic_history_times / (checkpoint_secs as usize);
-
-        let mut is_high_write_throughput = false;
-        let mut is_low_write_throughput = true;
-        if let Some(history) = table_write_throughput.get(table_id) {
-            if history.len() >= window_size {
-                is_high_write_throughput = history.iter().all(|throughput| {
-                    *throughput / checkpoint_secs > self.env.opts.table_write_throughput_threshold
-                });
-                is_low_write_throughput = history.iter().any(|throughput| {
-                    *throughput / checkpoint_secs < self.env.opts.min_table_split_write_throughput
-                });
-            }
-        }
-
-        let state_table_size = *table_size;
-
-        // 1. Avoid splitting a creating table
-        // 2. Avoid splitting a is_low_write_throughput creating table
-        // 3. Avoid splitting a non-high throughput medium-sized table
-        if is_creating_table
-            || (is_low_write_throughput)
-            || (state_table_size < self.env.opts.min_table_split_size && !is_high_write_throughput)
-        {
-            return;
-        }
-
-        // do not split a large table and a small table because it would increase IOPS
-        // of small table.
-        if parent_group_id != default_group_id && parent_group_id != mv_group_id {
-            let rest_group_size = group_size - state_table_size;
-            if rest_group_size < state_table_size
-                && rest_group_size < self.env.opts.min_table_split_size
-            {
-                return;
-            }
-        }
-
-        let ret = self
-            .move_state_tables_to_dedicated_compaction_group(
-                parent_group_id,
-                &[*table_id],
-                partition_vnode_count,
-            )
-            .await;
-        match ret {
-            Ok((target_compaction_group_id, cg_id_to_table_ids)) => {
-                tracing::info!(
-                    "split state table [{}] success (source_group_id {} target_group_id {})  cg_id_to_table_ids {:?}",
-                    table_id,
-                    parent_group_id,
-                    target_compaction_group_id,
-                    cg_id_to_table_ids
-                );
-            }
-            Err(e) => {
-                tracing::info!(
-                    error = %e.as_report(),
-                    "failed to move state table [{}] from group-{}",
-                    table_id,
-                    parent_group_id,
-                )
-            }
-        }
-    }
 }
 
 impl HummockManager {
@@ -402,7 +339,7 @@ impl HummockManager {
         split_table_ids: &[StateTableId],
         table_id_to_split: StateTableId,
         vnode_to_split: VirtualNode,
-        partition_vnode_count: u32,
+        partition_vnode_count: Option<u32>,
     ) -> Result<Vec<(CompactionGroupId, Vec<StateTableId>)>> {
         let mut result = vec![];
         let compaction_guard = self.compaction.write().await;
@@ -423,9 +360,12 @@ impl HummockManager {
         let member_table_ids = versioning
             .current_version
             .state_table_info
-            .compaction_group_member_table_ids(parent_group_id);
+            .compaction_group_member_table_ids(parent_group_id)
+            .iter()
+            .map(|table_id| table_id.table_id)
+            .collect::<BTreeSet<_>>();
 
-        if !member_table_ids.contains(&TableId::new(table_id_to_split)) {
+        if !member_table_ids.contains(&table_id_to_split) {
             return Err(Error::CompactionGroup(format!(
                 "table {} doesn't in group {}",
                 table_id_to_split, parent_group_id
@@ -435,10 +375,13 @@ impl HummockManager {
         let split_full_key = group_split::build_split_full_key(table_id_to_split, vnode_to_split);
 
         // change to vec for partition
-        let table_ids = member_table_ids
-            .iter()
-            .map(|table_id| table_id.table_id)
-            .collect_vec();
+        let table_ids = member_table_ids.into_iter().collect_vec();
+        if table_ids == split_table_ids {
+            return Err(Error::CompactionGroup(format!(
+                "invalid split attempt for group {}: all member tables are moved",
+                parent_group_id
+            )));
+        }
         // avoid decode split_key when caller is aware of the table_id and vnode
         let (table_ids_left, table_ids_right) =
             group_split::split_table_ids_with_table_id_and_vnode(
@@ -540,7 +483,10 @@ impl HummockManager {
             // The target `table_ids` might be split to an existing group, so we need to try to update its config
             for (cg_id, table_ids) in &result {
                 // check the split_tables had been place to the dedicated compaction group
-                if table_ids.len() == 1 && table_ids == split_table_ids {
+                if let Some(partition_vnode_count) = partition_vnode_count
+                    && table_ids.len() == 1
+                    && table_ids == split_table_ids
+                {
                     if let Err(err) = compaction_groups_txn.update_compaction_config(
                         &[*cg_id],
                         &[MutableConfig::SplitWeightByVnode(partition_vnode_count)],
@@ -615,7 +561,7 @@ impl HummockManager {
         &self,
         parent_group_id: CompactionGroupId,
         table_ids: &[StateTableId],
-        partition_vnode_count: u32,
+        partition_vnode_count: Option<u32>,
     ) -> Result<(
         CompactionGroupId,
         BTreeMap<CompactionGroupId, Vec<StateTableId>>,
@@ -698,14 +644,25 @@ impl HummockManager {
             )
             .await?;
         assert!(result_vec.len() <= 2);
-        for (cg_id, table_ids) in result_vec {
-            if table_ids.contains(&table_id_to_split) {
+
+        let mut finish_move = false;
+        for (cg_id, table_ids_after_split) in result_vec {
+            if table_ids_after_split.contains(&table_id_to_split) {
                 target_compaction_group_id = cg_id;
             }
-            cg_id_to_table_ids.insert(cg_id, table_ids);
+
+            if table_ids_after_split == table_ids {
+                finish_move = true;
+            }
+
+            cg_id_to_table_ids.insert(cg_id, table_ids_after_split);
         }
         check_table_ids_valid(&cg_id_to_table_ids);
 
+        if finish_move {
+            return Ok((target_compaction_group_id, cg_id_to_table_ids));
+        }
+
         // split 2
         // See the example above and the split rule in `split_compaction_group_impl`.
         let table_id_to_split = *table_ids.last().unwrap();
@@ -719,14 +676,372 @@ impl HummockManager {
             )
             .await?;
         assert!(result_vec.len() <= 2);
-        for (cg_id, table_ids) in result_vec {
-            if table_ids.contains(&table_id_to_split) {
+        for (cg_id, table_ids_after_split) in result_vec {
+            if table_ids_after_split.contains(&table_id_to_split) {
                 target_compaction_group_id = cg_id;
             }
-            cg_id_to_table_ids.insert(cg_id, table_ids);
+            cg_id_to_table_ids.insert(cg_id, table_ids_after_split);
         }
         check_table_ids_valid(&cg_id_to_table_ids);
 
         Ok((target_compaction_group_id, cg_id_to_table_ids))
     }
 }
+
+impl HummockManager {
+    /// Split the compaction group if the group is too large or contains high throughput tables.
+    pub async fn try_split_compaction_group(
+        &self,
+        table_write_throughput: &HashMap<u32, VecDeque<u64>>,
+        checkpoint_secs: u64,
+        group: CompactionGroupStatistic,
+    ) {
+        if group
+            .compaction_group_config
+            .compaction_config
+            .disable_auto_group_scheduling
+            .unwrap_or(false)
+        {
+            return;
+        }
+        // split high throughput table to dedicated compaction group
+        for (table_id, table_size) in &group.table_statistic {
+            self.try_move_high_throughput_table_to_dedicated_cg(
+                table_write_throughput,
+                table_id,
+                table_size,
+                checkpoint_secs,
+                group.group_id,
+            )
+            .await;
+        }
+
+        // split the huge group to multiple groups
+        self.try_split_huge_compaction_group(group).await;
+    }
+
+    /// Try to move the high throughput table to a dedicated compaction group.
+    pub async fn try_move_high_throughput_table_to_dedicated_cg(
+        &self,
+        table_write_throughput: &HashMap<u32, VecDeque<u64>>,
+        table_id: &u32,
+        _table_size: &u64,
+        checkpoint_secs: u64,
+        parent_group_id: u64,
+    ) {
+        if !table_write_throughput.contains_key(table_id) {
+            return;
+        }
+
+        let table_stat_throughput_sample_size =
+            ((self.env.opts.table_stat_throuput_window_seconds_for_split as f64)
+                / (checkpoint_secs as f64))
+                .ceil() as usize;
+        let table_throughput = table_write_throughput.get(table_id).unwrap();
+        if table_throughput.len() < table_stat_throughput_sample_size {
+            return;
+        }
+
+        let is_high_write_throughput = is_table_high_write_throughput(
+            table_throughput,
+            table_stat_throughput_sample_size,
+            self.env.opts.table_high_write_throughput_threshold,
+            self.env
+                .opts
+                .table_stat_high_write_throughput_ratio_for_split,
+        );
+
+        // do not split a table to dedicated compaction group if it is not high write throughput
+        if !is_high_write_throughput {
+            return;
+        }
+
+        let ret = self
+            .move_state_tables_to_dedicated_compaction_group(
+                parent_group_id,
+                &[*table_id],
+                Some(self.env.opts.partition_vnode_count),
+            )
+            .await;
+        match ret {
+            Ok(split_result) => {
+                tracing::info!("split state table [{}] from group-{} success table_vnode_partition_count {:?} split result {:?}", table_id, parent_group_id, self.env.opts.partition_vnode_count, split_result);
+            }
+            Err(e) => {
+                tracing::info!(
+                    error = %e.as_report(),
+                    "failed to split state table [{}] from group-{}",
+                    table_id,
+                    parent_group_id,
+                )
+            }
+        }
+    }
+
+    pub async fn try_split_huge_compaction_group(&self, group: CompactionGroupStatistic) {
+        let group_max_size = (group.compaction_group_config.max_estimated_group_size() as f64
+            * self.env.opts.split_group_size_ratio) as u64;
+        let is_huge_hybrid_group =
+            group.group_size > group_max_size && group.table_statistic.len() > 1; // avoid split single table group
+        if is_huge_hybrid_group {
+            let mut accumulated_size = 0;
+            let mut table_ids = Vec::default();
+            for (table_id, table_size) in &group.table_statistic {
+                accumulated_size += table_size;
+                table_ids.push(*table_id);
+                // split if the accumulated size is greater than half of the group size
+                // avoid split a small table to dedicated compaction group and trigger multiple merge
+                assert!(table_ids.is_sorted());
+                if accumulated_size * 2 > group_max_size {
+                    let ret = self
+                        .move_state_tables_to_dedicated_compaction_group(
+                            group.group_id,
+                            &table_ids,
+                            None,
+                        )
+                        .await;
+                    match ret {
+                        Ok(split_result) => {
+                            tracing::info!(
+                                "split_huge_compaction_group success {:?}",
+                                split_result
+                            );
+                            self.metrics
+                                .split_compaction_group_count
+                                .with_label_values(&[&group.group_id.to_string()])
+                                .inc();
+                            return;
+                        }
+                        Err(e) => {
+                            tracing::error!(
+                                error = %e.as_report(),
+                                "failed to split_huge_compaction_group table {:?} from group-{}",
+                                table_ids,
+                                group.group_id
+                            );
+
+                            return;
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    pub async fn try_merge_compaction_group(
+        &self,
+        table_write_throughput: &HashMap<u32, VecDeque<u64>>,
+        group: &CompactionGroupStatistic,
+        next_group: &CompactionGroupStatistic,
+        checkpoint_secs: u64,
+        created_tables: &HashSet<u32>,
+    ) -> Result<()> {
+        // TODO: remove this check after refactor group id
+        if group.group_id == StaticCompactionGroupId::StateDefault as u64
+            && next_group.group_id == StaticCompactionGroupId::MaterializedView as u64
+        {
+            return Err(Error::CompactionGroup(format!(
+                "group-{} and group-{} are both StaticCompactionGroupId",
+                group.group_id, next_group.group_id
+            )));
+        }
+
+        if group.table_statistic.is_empty() || next_group.table_statistic.is_empty() {
+            return Err(Error::CompactionGroup(format!(
+                "group-{} or group-{} is empty",
+                group.group_id, next_group.group_id
+            )));
+        }
+
+        if group
+            .compaction_group_config
+            .compaction_config
+            .disable_auto_group_scheduling
+            .unwrap_or(false)
+            || next_group
+                .compaction_group_config
+                .compaction_config
+                .disable_auto_group_scheduling
+                .unwrap_or(false)
+        {
+            return Err(Error::CompactionGroup(format!(
+                "group-{} or group-{} disable_auto_group_scheduling",
+                group.group_id, next_group.group_id
+            )));
+        }
+
+        // do not merge the compaction group which is creating
+        if check_is_creating_compaction_group(group, created_tables) {
+            return Err(Error::CompactionGroup(format!(
+                "Not Merge creating group {} next_group {}",
+                group.group_id, next_group.group_id
+            )));
+        }
+
+        // do not merge high throughput group
+        let table_stat_throughput_sample_size =
+            ((self.env.opts.table_stat_throuput_window_seconds_for_merge as f64)
+                / (checkpoint_secs as f64))
+                .ceil() as usize;
+
+        // merge the group which is low write throughput
+        if !check_is_low_write_throughput_compaction_group(
+            table_write_throughput,
+            table_stat_throughput_sample_size,
+            self.env.opts.table_low_write_throughput_threshold,
+            group,
+            self.env
+                .opts
+                .table_stat_low_write_throughput_ratio_for_merge,
+        ) {
+            return Err(Error::CompactionGroup(format!(
+                "Not Merge high throughput group {} next_group {}",
+                group.group_id, next_group.group_id
+            )));
+        }
+
+        let size_limit = (group.compaction_group_config.max_estimated_group_size() as f64
+            * self.env.opts.split_group_size_ratio) as u64;
+
+        if (group.group_size + next_group.group_size) > size_limit {
+            return Err(Error::CompactionGroup(format!(
+                "Not Merge huge group {} group_size {} next_group {} next_group_size {} size_limit {}",
+                group.group_id, group.group_size, next_group.group_id, next_group.group_size, size_limit
+            )));
+        }
+
+        if check_is_creating_compaction_group(next_group, created_tables) {
+            return Err(Error::CompactionGroup(format!(
+                "Not Merge creating group {} next group {}",
+                group.group_id, next_group.group_id
+            )));
+        }
+
+        if !check_is_low_write_throughput_compaction_group(
+            table_write_throughput,
+            table_stat_throughput_sample_size,
+            self.env.opts.table_low_write_throughput_threshold,
+            next_group,
+            self.env
+                .opts
+                .table_stat_low_write_throughput_ratio_for_merge,
+        ) {
+            return Err(Error::CompactionGroup(format!(
+                "Not Merge high throughput group {} next group {}",
+                group.group_id, next_group.group_id
+            )));
+        }
+
+        match self
+            .merge_compaction_group(group.group_id, next_group.group_id)
+            .await
+        {
+            Ok(()) => {
+                tracing::info!(
+                    "merge group-{} to group-{}",
+                    next_group.group_id,
+                    group.group_id,
+                );
+
+                self.metrics
+                    .merge_compaction_group_count
+                    .with_label_values(&[&group.group_id.to_string()])
+                    .inc();
+            }
+            Err(e) => {
+                tracing::info!(
+                    error = %e.as_report(),
+                    "failed to merge group-{} group-{}",
+                    next_group.group_id,
+                    group.group_id,
+                )
+            }
+        }
+
+        Ok(())
+    }
+}
+
+/// Check if the table is high write throughput with the given threshold and ratio.
+pub fn is_table_high_write_throughput(
+    table_throughput: &VecDeque<u64>,
+    sample_size: usize,
+    threshold: u64,
+    high_write_throughput_ratio: f64,
+) -> bool {
+    assert!(table_throughput.len() >= sample_size);
+    let mut high_write_throughput_count = 0;
+    for throughput in table_throughput
+        .iter()
+        .skip(table_throughput.len().saturating_sub(sample_size))
+    {
+        // only check the latest window_size
+        if *throughput > threshold {
+            high_write_throughput_count += 1;
+        }
+    }
+
+    high_write_throughput_count as f64 > sample_size as f64 * high_write_throughput_ratio
+}
+
+pub fn is_table_low_write_throughput(
+    table_throughput: &VecDeque<u64>,
+    sample_size: usize,
+    threshold: u64,
+    low_write_throughput_ratio: f64,
+) -> bool {
+    assert!(table_throughput.len() >= sample_size);
+
+    let mut low_write_throughput_count = 0;
+    for throughput in table_throughput
+        .iter()
+        .skip(table_throughput.len().saturating_sub(sample_size))
+    {
+        if *throughput <= threshold {
+            low_write_throughput_count += 1;
+        }
+    }
+
+    low_write_throughput_count as f64 > sample_size as f64 * low_write_throughput_ratio
+}
+
+fn check_is_low_write_throughput_compaction_group(
+    table_write_throughput: &HashMap<u32, VecDeque<u64>>,
+    sample_size: usize,
+    threshold: u64,
+    group: &CompactionGroupStatistic,
+    low_write_throughput_ratio: f64,
+) -> bool {
+    // check table exists
+    let live_table = group
+        .table_statistic
+        .keys()
+        .filter(|table_id| table_write_throughput.contains_key(table_id))
+        .filter(|table_id| table_write_throughput.get(table_id).unwrap().len() >= sample_size)
+        .cloned()
+        .collect_vec();
+
+    if live_table.is_empty() {
+        return false;
+    }
+
+    live_table.into_iter().all(|table_id| {
+        let table_write_throughput = table_write_throughput.get(&table_id).unwrap();
+        is_table_low_write_throughput(
+            table_write_throughput,
+            sample_size,
+            threshold,
+            low_write_throughput_ratio,
+        )
+    })
+}
+
+fn check_is_creating_compaction_group(
+    group: &CompactionGroupStatistic,
+    created_tables: &HashSet<u32>,
+) -> bool {
+    group
+        .table_statistic
+        .keys()
+        .any(|table_id| !created_tables.contains(table_id))
+}
diff --git a/src/meta/src/hummock/manager/compaction/mod.rs b/src/meta/src/hummock/manager/compaction/mod.rs
index 6ea718bc1e02e..cf2c448f10021 100644
--- a/src/meta/src/hummock/manager/compaction/mod.rs
+++ b/src/meta/src/hummock/manager/compaction/mod.rs
@@ -41,7 +41,7 @@ use rand::seq::SliceRandom;
 use rand::thread_rng;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_hummock_sdk::compact_task::{CompactTask, ReportTask};
-use risingwave_hummock_sdk::compaction_group::hummock_version_ext::HummockLevelsExt;
+use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::key_range::KeyRange;
 use risingwave_hummock_sdk::level::Levels;
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
@@ -88,6 +88,7 @@ use crate::hummock::manager::versioning::Versioning;
 use crate::hummock::metrics_utils::{
     build_compact_task_level_type_metrics_label, trigger_local_table_stat, trigger_sst_stat,
 };
+use crate::hummock::model::CompactionGroup;
 use crate::hummock::sequence::next_compaction_task_id;
 use crate::hummock::{commit_multi_var, start_measure_real_process_timer, HummockManager};
 use crate::manager::META_NODE_ID;
@@ -142,7 +143,7 @@ fn init_selectors() -> HashMap<compact_task::TaskType, Box<dyn CompactionSelecto
     compaction_selectors
 }
 
-impl<'a> HummockVersionTransaction<'a> {
+impl HummockVersionTransaction<'_> {
     fn apply_compact_task(&mut self, compact_task: &CompactTask) {
         let mut version_delta = self.new_delta();
         let trivial_move = CompactStatus::is_trivial_move_task(compact_task);
@@ -153,17 +154,15 @@ impl<'a> HummockVersionTransaction<'a> {
             .entry(compact_task.compaction_group_id)
             .or_default()
             .group_deltas;
-        let mut removed_table_ids_map: BTreeMap<u32, Vec<u64>> = BTreeMap::default();
+        let mut removed_table_ids_map: BTreeMap<u32, HashSet<u64>> = BTreeMap::default();
 
         for level in &compact_task.input_ssts {
             let level_idx = level.level_idx;
-            let mut removed_table_ids =
-                level.table_infos.iter().map(|sst| sst.sst_id).collect_vec();
 
             removed_table_ids_map
                 .entry(level_idx)
                 .or_default()
-                .append(&mut removed_table_ids);
+                .extend(level.table_infos.iter().map(|sst| sst.sst_id));
         }
 
         for (level_idx, removed_table_ids) in removed_table_ids_map {
@@ -181,7 +180,7 @@ impl<'a> HummockVersionTransaction<'a> {
         let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new(
             compact_task.target_level,
             compact_task.target_sub_level_id,
-            vec![], // default
+            HashSet::new(), // default
             compact_task.sorted_output_ssts.clone(),
             compact_task.split_weight_by_vnode,
         ));
@@ -1098,7 +1097,6 @@ impl HummockManager {
     ///
     /// Return Ok(false) indicates either the task is not found,
     /// or the task is not owned by `context_id` when `context_id` is not None.
-
     pub async fn report_compact_tasks(&self, report_tasks: Vec<ReportTask>) -> Result<Vec<bool>> {
         let mut guard = self.compaction.write().await;
         let deterministic_mode = self.env.opts.compaction_deterministic_test;
@@ -1160,6 +1158,11 @@ impl HummockManager {
                     compact_status.report_compact_task(&compact_task);
                 }
                 None => {
+                    // When the group_id is not found in the compaction_statuses, it means the group has been removed.
+                    // The task is invalid and should be canceled.
+                    // e.g.
+                    // 1. The group is removed by the user unregistering the tables
+                    // 2. The group is removed by the group scheduling algorithm
                     compact_task.task_status = TaskStatus::InvalidGroupCanceled;
                 }
             }
@@ -1174,6 +1177,7 @@ impl HummockManager {
                 .iter()
                 .map(|level| level.level_idx)
                 .collect();
+
             let is_success = if let TaskStatus::Success = compact_task.task_status {
                 if let Err(e) = self
                     .report_compaction_sanity_check(&task.object_timestamps)
@@ -1200,6 +1204,7 @@ impl HummockManager {
                             compact_task_to_string(&compact_task)
                         );
                     }
+
                     input_exist
                 }
             } else {
@@ -1480,7 +1485,7 @@ impl HummockManager {
                         .table_vnode_partition
                         .insert(table_id, default_partition_count);
                 } else if (compact_table_size > compact_task_table_size_partition_threshold_low
-                    || (write_throughput > self.env.opts.table_write_throughput_threshold
+                    || (write_throughput > self.env.opts.table_high_write_throughput_threshold
                         && compact_table_size > compaction_config.target_file_size_base))
                     && hybrid_vnode_count > 0
                 {
@@ -1661,3 +1666,11 @@ impl Compaction {
             .collect()
     }
 }
+
+#[derive(Clone, Default)]
+pub struct CompactionGroupStatistic {
+    pub group_id: CompactionGroupId,
+    pub group_size: u64,
+    pub table_statistic: BTreeMap<StateTableId, u64>,
+    pub compaction_group_config: CompactionGroup,
+}
diff --git a/src/meta/src/hummock/manager/context.rs b/src/meta/src/hummock/manager/context.rs
index 9b1fa246451eb..6256c09ce75a7 100644
--- a/src/meta/src/hummock/manager/context.rs
+++ b/src/meta/src/hummock/manager/context.rs
@@ -15,6 +15,7 @@
 use std::collections::{BTreeMap, HashMap, HashSet};
 
 use fail::fail_point;
+use futures::{stream, StreamExt};
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_hummock_sdk::version::HummockVersion;
@@ -22,7 +23,9 @@ use risingwave_hummock_sdk::{
     HummockContextId, HummockSstableObjectId, HummockVersionId, LocalSstableInfo,
     INVALID_VERSION_ID,
 };
+use risingwave_meta_model::hummock_gc_history;
 use risingwave_pb::hummock::{HummockPinnedVersion, ValidationTask};
+use sea_orm::{DatabaseConnection, EntityTrait};
 
 use crate::controller::SqlMetaStore;
 use crate::hummock::error::{Error, Result};
@@ -241,6 +244,10 @@ impl HummockManager {
                     .iter()
                     .map(|s| (s.sst_info.object_id, s.created_at)),
             )?;
+            if self.env.opts.gc_history_retention_time_sec != 0 {
+                let ids = sstables.iter().map(|s| s.sst_info.object_id).collect_vec();
+                check_gc_history(&self.meta_store_ref().conn, ids).await?;
+            }
         }
 
         async {
@@ -292,7 +299,12 @@ impl HummockManager {
             now,
             self.env.opts.min_sst_retention_time_sec,
             object_timestamps.iter().map(|(k, v)| (*k, *v)),
-        )
+        )?;
+        if self.env.opts.gc_history_retention_time_sec != 0 {
+            let ids = object_timestamps.iter().map(|(id, _)| *id).collect_vec();
+            check_gc_history(&self.meta_store_ref().conn, ids).await?;
+        }
+        Ok(())
     }
 }
 
@@ -310,6 +322,34 @@ fn check_sst_retention(
     Ok(())
 }
 
+async fn check_gc_history(
+    db: &DatabaseConnection,
+    // need IntoIterator to work around stream's "implementation of `std::iter::Iterator` is not general enough" error.
+    object_ids: impl IntoIterator<Item = HummockSstableObjectId>,
+) -> Result<()> {
+    let futures = object_ids.into_iter().map(|id| async move {
+        let id: risingwave_meta_model::HummockSstableObjectId = id.try_into().unwrap();
+        hummock_gc_history::Entity::find_by_id(id)
+            .one(db)
+            .await
+            .map_err(Error::from)
+    });
+    let res: Vec<_> = stream::iter(futures).buffer_unordered(10).collect().await;
+    let res: Result<Vec<_>> = res.into_iter().collect();
+    let mut expired_object_ids = res?.into_iter().flatten().peekable();
+    if expired_object_ids.peek().is_none() {
+        return Ok(());
+    }
+    let expired_object_ids: Vec<_> = expired_object_ids.collect();
+    tracing::error!(
+        ?expired_object_ids,
+        "new SSTs are rejected because they have already been GCed"
+    );
+    Err(Error::InvalidSst(
+        expired_object_ids[0].object_id as HummockSstableObjectId,
+    ))
+}
+
 // pin and unpin method
 impl HummockManager {
     /// Pin the current greatest hummock version. The pin belongs to `context_id`
diff --git a/src/meta/src/hummock/manager/gc.rs b/src/meta/src/hummock/manager/gc.rs
index f84404d322c77..4187ce818b345 100644
--- a/src/meta/src/hummock/manager/gc.rs
+++ b/src/meta/src/hummock/manager/gc.rs
@@ -18,18 +18,19 @@ use std::ops::Bound::{Excluded, Included};
 use std::ops::DerefMut;
 use std::time::{Duration, SystemTime};
 
+use chrono::DateTime;
 use futures::future::try_join_all;
 use itertools::Itertools;
 use parking_lot::Mutex;
 use risingwave_hummock_sdk::HummockSstableObjectId;
-use risingwave_meta_model::hummock_sequence;
 use risingwave_meta_model::hummock_sequence::HUMMOCK_NOW;
+use risingwave_meta_model::{hummock_gc_history, hummock_sequence};
 use risingwave_meta_model_migration::OnConflict;
 use risingwave_pb::hummock::subscribe_compaction_event_response::Event as ResponseEvent;
 use risingwave_pb::hummock::FullScanTask;
 use risingwave_pb::stream_service::GetMinUncommittedSstIdRequest;
 use risingwave_rpc_client::StreamClientPool;
-use sea_orm::{ActiveValue, EntityTrait};
+use sea_orm::{ActiveValue, ColumnTrait, EntityTrait, QueryFilter, Set};
 
 use crate::hummock::error::{Error, Result};
 use crate::hummock::manager::commit_multi_var;
@@ -137,7 +138,7 @@ impl HummockManager {
     pub async fn extend_objects_to_delete_from_scan(
         &self,
         object_ids: &[HummockSstableObjectId],
-    ) -> usize {
+    ) -> Result<usize> {
         let tracked_object_ids: HashSet<HummockSstableObjectId> = {
             let versioning = self.versioning.read().await;
             let context_info = self.context_info.read().await;
@@ -167,13 +168,15 @@ impl HummockManager {
         let to_delete = object_ids
             .iter()
             .filter(|object_id| !tracked_object_ids.contains(object_id))
+            .copied()
             .collect_vec();
+        let to_delete_num = to_delete.len();
+        self.write_gc_history(to_delete.iter().cloned()).await?;
         // This lock ensures that during commit_epoch or report_compact_tasks, where versioning lock is held,
         // no new objects will be marked for deletion here.
         let _versioning = self.versioning.read().await;
-        self.delete_object_tracker
-            .add(to_delete.iter().map(|id| **id));
-        to_delete.len()
+        self.delete_object_tracker.add(to_delete.into_iter());
+        Ok(to_delete_num)
     }
 
     /// Starts a full GC.
@@ -232,6 +235,7 @@ impl HummockManager {
         &self,
         object_ids: Vec<HummockSstableObjectId>,
         next_start_after: Option<String>,
+        pinned_by_metadata_backup: HashSet<HummockSstableObjectId>,
     ) -> Result<usize> {
         // It's crucial to collect_min_uncommitted_sst_id (i.e. `min_sst_id`) only after LIST object store (i.e. `object_ids`).
         // Because after getting `min_sst_id`, new compute nodes may join and generate new uncommitted SSTs that are not covered by `min_sst_id`.
@@ -268,8 +272,14 @@ impl HummockManager {
             .filter(|s| !pinned_object_ids.contains(s))
             .collect_vec();
         let after_time_travel = object_ids.len();
+        // filter by metadata backup
+        let object_ids = object_ids
+            .into_iter()
+            .filter(|s| !pinned_by_metadata_backup.contains(s))
+            .collect_vec();
+        let after_metadata_backup = object_ids.len();
         // filter by version
-        let after_version = self.extend_objects_to_delete_from_scan(&object_ids).await;
+        let after_version = self.extend_objects_to_delete_from_scan(&object_ids).await?;
         metrics
             .full_gc_selected_object_count
             .observe(after_version as _);
@@ -277,6 +287,7 @@ impl HummockManager {
             candidate_object_number,
             after_min_sst_id,
             after_time_travel,
+            after_metadata_backup,
             after_version,
             "complete full gc"
         );
@@ -347,6 +358,38 @@ impl HummockManager {
             self.metrics.total_object_count.set(total_object_count as _);
         }
     }
+
+    async fn write_gc_history(
+        &self,
+        object_ids: impl Iterator<Item = HummockSstableObjectId>,
+    ) -> Result<()> {
+        if self.env.opts.gc_history_retention_time_sec == 0 {
+            return Ok(());
+        }
+        let now = self.now().await?;
+        let dt = DateTime::from_timestamp(now.try_into().unwrap(), 0).unwrap();
+        let models = object_ids.map(|o| hummock_gc_history::ActiveModel {
+            object_id: Set(o.try_into().unwrap()),
+            mark_delete_at: Set(dt.naive_utc()),
+        });
+        let db = &self.meta_store_ref().conn;
+        let gc_history_low_watermark = DateTime::from_timestamp(
+            now.saturating_sub(self.env.opts.gc_history_retention_time_sec)
+                .try_into()
+                .unwrap(),
+            0,
+        )
+        .unwrap();
+        hummock_gc_history::Entity::delete_many()
+            .filter(hummock_gc_history::Column::MarkDeleteAt.lt(gc_history_low_watermark))
+            .exec(db)
+            .await?;
+        hummock_gc_history::Entity::insert_many(models)
+            .on_conflict_do_nothing()
+            .exec(db)
+            .await?;
+        Ok(())
+    }
 }
 
 async fn collect_min_uncommitted_sst_id(
@@ -475,6 +518,7 @@ impl PagedMetric {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
     use std::sync::Arc;
     use std::time::Duration;
 
@@ -536,15 +580,20 @@ mod tests {
 
         // Empty input results immediate return, without waiting heartbeat.
         hummock_manager
-            .complete_full_gc(vec![], None)
+            .complete_full_gc(vec![], None, HashSet::default())
             .await
             .unwrap();
 
         // LSMtree is empty. All input SST ids should be treated as garbage.
+        // Use fake object ids, because they'll be written to GC history and they shouldn't affect later commit.
         assert_eq!(
             3,
             hummock_manager
-                .complete_full_gc(vec![1, 2, 3], None)
+                .complete_full_gc(
+                    vec![i64::MAX as u64 - 2, i64::MAX as u64 - 1, i64::MAX as u64],
+                    None,
+                    HashSet::new(),
+                )
                 .await
                 .unwrap()
         );
@@ -569,7 +618,8 @@ mod tests {
             hummock_manager
                 .complete_full_gc(
                     [committed_object_ids, vec![max_committed_object_id + 1]].concat(),
-                    None
+                    None,
+                    HashSet::default(),
                 )
                 .await
                 .unwrap()
diff --git a/src/meta/src/hummock/manager/mod.rs b/src/meta/src/hummock/manager/mod.rs
index 8aef8a5e45ba8..68479d64727a2 100644
--- a/src/meta/src/hummock/manager/mod.rs
+++ b/src/meta/src/hummock/manager/mod.rs
@@ -410,11 +410,6 @@ impl HummockManager {
             .collect();
 
         self.delete_object_tracker.clear();
-        // Not delete stale objects when archive or time travel is enabled
-        if !self.env.opts.enable_hummock_data_archive && !self.time_travel_enabled().await {
-            versioning_guard.mark_objects_for_deletion(context_info, &self.delete_object_tracker);
-        }
-
         self.initial_compaction_group_config_after_load(
             versioning_guard,
             self.compaction_group_manager.write().await.deref_mut(),
diff --git a/src/meta/src/hummock/manager/tests.rs b/src/meta/src/hummock/manager/tests.rs
index 8077998aa1ee0..d78f0bc163409 100644
--- a/src/meta/src/hummock/manager/tests.rs
+++ b/src/meta/src/hummock/manager/tests.rs
@@ -1192,7 +1192,8 @@ async fn test_extend_objects_to_delete() {
     assert_eq!(
         hummock_manager
             .extend_objects_to_delete_from_scan(&all_object_ids)
-            .await,
+            .await
+            .unwrap(),
         orphan_sst_num as usize
     );
     assert_eq!(
@@ -1213,7 +1214,8 @@ async fn test_extend_objects_to_delete() {
     assert_eq!(
         hummock_manager
             .extend_objects_to_delete_from_scan(&all_object_ids)
-            .await,
+            .await
+            .unwrap(),
         orphan_sst_num as usize
     );
     let objects_to_delete = hummock_manager.get_objects_to_delete();
@@ -1242,7 +1244,8 @@ async fn test_extend_objects_to_delete() {
     assert_eq!(
         hummock_manager
             .extend_objects_to_delete_from_scan(&all_object_ids)
-            .await,
+            .await
+            .unwrap(),
         orphan_sst_num as usize
     );
     let objects_to_delete = hummock_manager.get_objects_to_delete();
@@ -1270,7 +1273,8 @@ async fn test_extend_objects_to_delete() {
     assert_eq!(
         hummock_manager
             .extend_objects_to_delete_from_scan(&all_object_ids)
-            .await,
+            .await
+            .unwrap(),
         orphan_sst_num as usize + 3
     );
     let objects_to_delete = hummock_manager.get_objects_to_delete();
@@ -1498,13 +1502,13 @@ async fn test_move_state_tables_to_dedicated_compaction_group_on_demand_basic()
     assert_eq!(original_groups, vec![2, 3]);
 
     let err = hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(100, &[0], 0)
+        .move_state_tables_to_dedicated_compaction_group(100, &[0], None)
         .await
         .unwrap_err();
     assert_eq!("compaction group error: invalid group 100", err.to_string());
 
     let err = hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(2, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(2, &[100], None)
         .await
         .unwrap_err();
     assert_eq!(
@@ -1540,7 +1544,7 @@ async fn test_move_state_tables_to_dedicated_compaction_group_on_demand_basic()
 
     let compaction_group_id = StaticCompactionGroupId::StateDefault.into();
     let err = hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100, 101], 0)
+        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100, 101], None)
         .await
         .unwrap_err();
     assert_eq!(
@@ -1556,7 +1560,7 @@ async fn test_move_state_tables_to_dedicated_compaction_group_on_demand_basic()
         .unwrap();
 
     hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100, 101], 0)
+        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100, 101], None)
         .await
         .unwrap();
     let current_version = hummock_manager.get_current_version().await;
@@ -1625,7 +1629,7 @@ async fn test_move_state_tables_to_dedicated_compaction_group_on_demand_non_triv
 
     let compaction_group_id = StaticCompactionGroupId::StateDefault.into();
     hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], None)
         .await
         .unwrap();
 
@@ -1727,7 +1731,7 @@ async fn test_move_state_tables_to_dedicated_compaction_group_trivial_expired()
         .unwrap();
 
     hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], None)
         .await
         .unwrap();
 
@@ -1899,7 +1903,7 @@ async fn test_move_state_tables_to_dedicated_compaction_group_on_demand_bottom_l
     let base_level: usize = 6;
 
     hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(2, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(2, &[100], None)
         .await
         .unwrap();
 
@@ -2000,7 +2004,7 @@ async fn test_compaction_task_expiration_due_to_split_group() {
         get_manual_compact_task(hummock_manager.clone(), compaction_group_id, 0).await;
     assert_eq!(compaction_task.input_ssts[0].table_infos.len(), 2);
     hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], None)
         .await
         .unwrap();
 
@@ -2095,7 +2099,7 @@ async fn test_move_tables_between_compaction_group() {
     assert!(sst_ids.contains(&14));
 
     hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(2, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(2, &[100], None)
         .await
         .unwrap();
     let compaction_group_id =
@@ -2234,7 +2238,7 @@ async fn test_partition_level() {
         .move_state_tables_to_dedicated_compaction_group(
             compaction_group_id,
             &[100],
-            env.opts.partition_vnode_count,
+            Some(env.opts.partition_vnode_count),
         )
         .await
         .unwrap();
@@ -2355,7 +2359,7 @@ async fn test_unregister_moved_table() {
 
     let compaction_group_id = StaticCompactionGroupId::StateDefault.into();
     let new_compaction_group_id = hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], None)
         .await
         .unwrap()
         .0;
@@ -2483,7 +2487,7 @@ async fn test_merge_compaction_group_task_expired() {
 
     // will cancel the task
     hummock_manager
-        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], 0)
+        .move_state_tables_to_dedicated_compaction_group(compaction_group_id, &[100], None)
         .await
         .unwrap();
 
@@ -2526,9 +2530,15 @@ async fn test_merge_compaction_group_task_expired() {
         .unwrap()
         .unwrap();
 
+    let created_tables = HashSet::from_iter(vec![100, 101, 102]);
+
     // will cancel the task2
     hummock_manager
-        .merge_compaction_group(left_compaction_group_id, right_compaction_group_id)
+        .merge_compaction_group_for_test(
+            left_compaction_group_id,
+            right_compaction_group_id,
+            created_tables,
+        )
         .await
         .unwrap();
 
diff --git a/src/meta/src/hummock/manager/time_travel.rs b/src/meta/src/hummock/manager/time_travel.rs
index d342ec5b7bc61..1ceaad5cfd390 100644
--- a/src/meta/src/hummock/manager/time_travel.rs
+++ b/src/meta/src/hummock/manager/time_travel.rs
@@ -17,14 +17,13 @@ use std::collections::{HashMap, HashSet, VecDeque};
 use anyhow::anyhow;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
-use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::util::epoch::Epoch;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::time_travel::{
     refill_version, IncompleteHummockVersion, IncompleteHummockVersionDelta,
 };
-use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
+use risingwave_hummock_sdk::version::{GroupDeltaCommon, HummockVersion, HummockVersionDelta};
 use risingwave_hummock_sdk::{
     CompactionGroupId, HummockEpoch, HummockSstableId, HummockSstableObjectId,
 };
@@ -34,7 +33,6 @@ use risingwave_meta_model::{
     hummock_time_travel_version,
 };
 use risingwave_pb::hummock::{PbHummockVersion, PbHummockVersionDelta};
-use sea_orm::sea_query::OnConflict;
 use sea_orm::ActiveValue::Set;
 use sea_orm::{
     ColumnTrait, Condition, DatabaseTransaction, EntityTrait, QueryFilter, QueryOrder, QuerySelect,
@@ -46,14 +44,6 @@ use crate::hummock::HummockManager;
 
 /// Time travel.
 impl HummockManager {
-    pub(crate) async fn time_travel_enabled(&self) -> bool {
-        self.env
-            .system_params_reader()
-            .await
-            .time_travel_retention_ms()
-            > 0
-    }
-
     pub(crate) async fn init_time_travel_state(&self) -> Result<()> {
         let sql_store = self.env.meta_store_ref();
         let mut guard = self.versioning.write().await;
@@ -383,12 +373,7 @@ impl HummockManager {
                     sstable_info: Set(SstableInfoV2Backend::from(&sst_info.to_protobuf())),
                 };
                 hummock_sstable_info::Entity::insert(m)
-                    .on_conflict(
-                        OnConflict::column(hummock_sstable_info::Column::SstId)
-                            .do_nothing()
-                            .to_owned(),
-                    )
-                    .do_nothing()
+                    .on_conflict_do_nothing()
                     .exec(txn)
                     .await?;
                 count += 1;
@@ -437,18 +422,13 @@ impl HummockManager {
                     .into()),
             };
             hummock_time_travel_version::Entity::insert(m)
-                .on_conflict(
-                    OnConflict::column(hummock_time_travel_version::Column::VersionId)
-                        .do_nothing()
-                        .to_owned(),
-                )
-                .do_nothing()
+                .on_conflict_do_nothing()
                 .exec(txn)
                 .await?;
         }
         let written = write_sstable_infos(
             delta
-                .newly_added_sst_infos(&select_groups)
+                .newly_added_sst_infos(Some(&select_groups))
                 .filter(|s| !skip_sst_ids.contains(&s.sst_id)),
             txn,
         )
@@ -468,12 +448,7 @@ impl HummockManager {
                     .into()),
             };
             hummock_time_travel_delta::Entity::insert(m)
-                .on_conflict(
-                    OnConflict::column(hummock_time_travel_delta::Column::VersionId)
-                        .do_nothing()
-                        .to_owned(),
-                )
-                .do_nothing()
+                .on_conflict_do_nothing()
                 .exec(txn)
                 .await?;
         }
@@ -489,6 +464,11 @@ fn replay_archive(
     let mut last_version = HummockVersion::from_persisted_protobuf(&version);
     for d in deltas {
         let d = HummockVersionDelta::from_persisted_protobuf(&d);
+        debug_assert!(
+            !should_mark_next_time_travel_version_snapshot(&d),
+            "unexpected time travel delta {:?}",
+            d
+        );
         // Need to work around the assertion in `apply_version_delta`.
         // Because compaction deltas are not included in time travel archive.
         while last_version.id < d.prev_id {
@@ -522,3 +502,13 @@ fn should_ignore_group(root_group_id: CompactionGroupId) -> bool {
 pub fn require_sql_meta_store_err() -> Error {
     Error::TimeTravel(anyhow!("require SQL meta store"))
 }
+
+/// Time travel delta replay only expect `NewL0SubLevel`. In all other cases, a new version snapshot should be created.
+pub fn should_mark_next_time_travel_version_snapshot(delta: &HummockVersionDelta) -> bool {
+    delta.group_deltas.iter().any(|(_, deltas)| {
+        deltas
+            .group_deltas
+            .iter()
+            .any(|d| !matches!(d, GroupDeltaCommon::NewL0SubLevel(_)))
+    })
+}
diff --git a/src/meta/src/hummock/manager/timer_task.rs b/src/meta/src/hummock/manager/timer_task.rs
index 0cefe157d8493..5b62ba883e7d7 100644
--- a/src/meta/src/hummock/manager/timer_task.rs
+++ b/src/meta/src/hummock/manager/timer_task.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -43,7 +43,7 @@ impl HummockManager {
             const COMPACTION_HEARTBEAT_PERIOD_SEC: u64 = 1;
 
             pub enum HummockTimerEvent {
-                GroupSchedule,
+                GroupScheduleSplit,
                 CheckDeadTask,
                 Report,
                 CompactionHeartBeatExpiredCheck,
@@ -54,6 +54,8 @@ impl HummockManager {
                 TombstoneCompactionTrigger,
 
                 FullGc,
+
+                GroupScheduleMerge,
             }
             let mut check_compact_trigger_interval =
                 tokio::time::interval(Duration::from_secs(CHECK_PENDING_TASK_PERIOD_SEC));
@@ -145,22 +147,38 @@ impl HummockManager {
                 Box::pin(tombstone_reclaim_trigger),
             ];
 
-            let periodic_scheduling_compaction_group_interval_sec = hummock_manager
+            let periodic_scheduling_compaction_group_split_interval_sec = hummock_manager
                 .env
                 .opts
-                .periodic_scheduling_compaction_group_interval_sec;
+                .periodic_scheduling_compaction_group_split_interval_sec;
 
-            if periodic_scheduling_compaction_group_interval_sec > 0 {
+            if periodic_scheduling_compaction_group_split_interval_sec > 0 {
                 let mut scheduling_compaction_group_trigger_interval = tokio::time::interval(
-                    Duration::from_secs(periodic_scheduling_compaction_group_interval_sec),
+                    Duration::from_secs(periodic_scheduling_compaction_group_split_interval_sec),
                 );
                 scheduling_compaction_group_trigger_interval
                     .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
-
-                let group_scheduling_trigger =
+                let group_scheduling_split_trigger =
                     IntervalStream::new(scheduling_compaction_group_trigger_interval)
-                        .map(|_| HummockTimerEvent::GroupSchedule);
-                triggers.push(Box::pin(group_scheduling_trigger));
+                        .map(|_| HummockTimerEvent::GroupScheduleSplit);
+                triggers.push(Box::pin(group_scheduling_split_trigger));
+            }
+
+            let periodic_scheduling_compaction_group_merge_interval_sec = hummock_manager
+                .env
+                .opts
+                .periodic_scheduling_compaction_group_merge_interval_sec;
+
+            if periodic_scheduling_compaction_group_merge_interval_sec > 0 {
+                let mut scheduling_compaction_group_merge_trigger_interval = tokio::time::interval(
+                    Duration::from_secs(periodic_scheduling_compaction_group_merge_interval_sec),
+                );
+                scheduling_compaction_group_merge_trigger_interval
+                    .set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
+                let group_scheduling_merge_trigger =
+                    IntervalStream::new(scheduling_compaction_group_merge_trigger_interval)
+                        .map(|_| HummockTimerEvent::GroupScheduleMerge);
+                triggers.push(Box::pin(group_scheduling_merge_trigger));
             }
 
             let event_stream = select_all(triggers);
@@ -170,8 +188,8 @@ impl HummockManager {
             let shutdown_rx_shared = shutdown_rx.shared();
 
             tracing::info!(
-                "Hummock timer task [GroupScheduling interval {} sec] [CheckDeadTask interval {} sec] [Report interval {} sec] [CompactionHeartBeat interval {} sec]",
-                periodic_scheduling_compaction_group_interval_sec, CHECK_PENDING_TASK_PERIOD_SEC, STAT_REPORT_PERIOD_SEC, COMPACTION_HEARTBEAT_PERIOD_SEC
+                "Hummock timer task [GroupSchedulingSplit interval {} sec] [GroupSchedulingMerge interval {} sec] [CheckDeadTask interval {} sec] [Report interval {} sec] [CompactionHeartBeat interval {} sec]",
+                periodic_scheduling_compaction_group_split_interval_sec, periodic_scheduling_compaction_group_merge_interval_sec, CHECK_PENDING_TASK_PERIOD_SEC, STAT_REPORT_PERIOD_SEC, COMPACTION_HEARTBEAT_PERIOD_SEC
             );
 
             loop {
@@ -190,12 +208,20 @@ impl HummockManager {
                                     hummock_manager.check_dead_task().await;
                                 }
 
-                                HummockTimerEvent::GroupSchedule => {
+                                HummockTimerEvent::GroupScheduleSplit => {
                                     if hummock_manager.env.opts.compaction_deterministic_test {
                                         continue;
                                     }
 
-                                    hummock_manager.on_handle_schedule_group().await;
+                                    hummock_manager.on_handle_schedule_group_split().await;
+                                }
+
+                                HummockTimerEvent::GroupScheduleMerge => {
+                                    if hummock_manager.env.opts.compaction_deterministic_test {
+                                        continue;
+                                    }
+
+                                    hummock_manager.on_handle_schedule_group_merge().await;
                                 }
 
                                 HummockTimerEvent::Report => {
@@ -243,6 +269,71 @@ impl HummockManager {
                                             compaction_group_config.group_id(),
                                         )
                                     }
+
+                                    {
+                                        let group_infos = hummock_manager
+                                            .calculate_compaction_group_statistic()
+                                            .await;
+                                        let compaction_group_count = group_infos.len();
+                                        hummock_manager
+                                            .metrics
+                                            .compaction_group_count
+                                            .set(compaction_group_count as i64);
+
+                                        let tables_throughput =
+                                            hummock_manager.history_table_throughput.read().clone();
+
+                                        let current_version_levels = &hummock_manager
+                                            .versioning
+                                            .read()
+                                            .await
+                                            .current_version
+                                            .levels;
+
+                                        for group_info in group_infos {
+                                            hummock_manager
+                                                .metrics
+                                                .compaction_group_size
+                                                .with_label_values(&[&group_info
+                                                    .group_id
+                                                    .to_string()])
+                                                .set(group_info.group_size as _);
+                                            // accumulate the throughput of all tables in the group
+                                            let mut avg_throuput = 0;
+                                            for table_id in group_info.table_statistic.keys() {
+                                                if let Some(throuput) =
+                                                    tables_throughput.get(table_id)
+                                                {
+                                                    let table_avg_throughput =
+                                                        throuput.iter().sum::<u64>()
+                                                            / throuput.len() as u64;
+
+                                                    avg_throuput += table_avg_throughput;
+                                                }
+                                            }
+
+                                            hummock_manager
+                                                .metrics
+                                                .compaction_group_throughput
+                                                .with_label_values(&[&group_info
+                                                    .group_id
+                                                    .to_string()])
+                                                .set(avg_throuput as _);
+
+                                            if let Some(group_levels) =
+                                                current_version_levels.get(&group_info.group_id)
+                                            {
+                                                let file_count = group_levels.count_ssts();
+                                                hummock_manager
+                                                    .metrics
+                                                    .compaction_group_file_count
+                                                    .with_label_values(&[&group_info
+                                                        .group_id
+                                                        .to_string()])
+                                                    .set(file_count as _);
+                                            }
+                                        }
+                                    }
                                 }
 
                                 HummockTimerEvent::CompactionHeartBeatExpiredCheck => {
@@ -438,55 +529,37 @@ impl HummockManager {
         }
     }
 
-    /// * For compaction group with only one single state-table, do not change it again.
-    /// * For state-table which only write less than `HISTORY_TABLE_INFO_WINDOW_SIZE` times, do not
-    ///   change it. Because we need more statistic data to decide split strategy.
-    /// * For state-table with low throughput which write no more than
-    ///   `min_table_split_write_throughput` data, never split it.
-    /// * For state-table whose size less than `min_table_split_size`, do not split it unless its
-    ///   throughput keep larger than `table_write_throughput_threshold` for a long time.
-    /// * For state-table whose throughput less than `min_table_split_write_throughput`, do not
-    ///   increase it size of base-level.
-    async fn on_handle_schedule_group(&self) {
+    /// Try to schedule a compaction `split` for the given compaction groups.
+    /// The `split` will be triggered if the following conditions are met:
+    /// 1. `state table throughput`: If the table is in a high throughput state and it belongs to a multi table group, then an attempt will be made to split the table into separate compaction groups to increase its throughput and reduce the impact on write amplification.
+    /// 2. `group size`: If the group size has exceeded the set upper limit, e.g. `max_group_size` * `split_group_size_ratio`
+    async fn on_handle_schedule_group_split(&self) {
         let params = self.env.system_params_reader().await;
         let barrier_interval_ms = params.barrier_interval_ms() as u64;
         let checkpoint_secs = std::cmp::max(
             1,
             params.checkpoint_frequency() * barrier_interval_ms / 1000,
         );
-        let created_tables = match self.metadata_manager.get_created_table_ids().await {
-            Ok(created_tables) => created_tables,
-            Err(err) => {
-                tracing::warn!(error = %err.as_report(), "failed to fetch created table ids");
-                return;
-            }
-        };
-        let created_tables: HashSet<u32> = HashSet::from_iter(created_tables);
         let table_write_throughput = self.history_table_throughput.read().clone();
         let mut group_infos = self.calculate_compaction_group_statistic().await;
         group_infos.sort_by_key(|group| group.group_size);
         group_infos.reverse();
 
-        for group in &group_infos {
+        for group in group_infos {
             if group.table_statistic.len() == 1 {
                 // no need to handle the separate compaciton group
                 continue;
             }
 
-            self.try_split_compaction_group(
-                &table_write_throughput,
-                checkpoint_secs,
-                group,
-                &created_tables,
-            )
-            .await;
+            self.try_split_compaction_group(&table_write_throughput, checkpoint_secs, group)
+                .await;
         }
     }
 
     async fn on_handle_trigger_multi_group(&self, task_type: compact_task::TaskType) {
         for cg_id in self.compaction_group_ids().await {
             if let Err(e) = self.compaction_state.try_sched_compaction(cg_id, task_type) {
-                tracing::warn!(
+                tracing::error!(
                     error = %e.as_report(),
                     "Failed to schedule {:?} compaction for compaction group {}",
                     task_type,
@@ -495,4 +568,72 @@ impl HummockManager {
             }
         }
     }
+
+    /// Try to schedule a compaction merge for the given compaction groups.
+    /// The merge will be triggered if the following conditions are met:
+    /// 1. The compaction group is not contains creating table.
+    /// 2. The compaction group is a small group.
+    /// 3. All tables in compaction group is in a low throughput state.
+    async fn on_handle_schedule_group_merge(&self) {
+        let created_tables = match self.metadata_manager.get_created_table_ids().await {
+            Ok(created_tables) => HashSet::from_iter(created_tables),
+            Err(err) => {
+                tracing::warn!(error = %err.as_report(), "failed to fetch created table ids");
+                return;
+            }
+        };
+        let table_write_throughput = self.history_table_throughput.read().clone();
+        let mut group_infos = self.calculate_compaction_group_statistic().await;
+        // sort by first table id for deterministic merge order
+        group_infos.sort_by_key(|group| {
+            let table_ids = group
+                .table_statistic
+                .keys()
+                .cloned()
+                .collect::<BTreeSet<_>>();
+            table_ids.iter().next().cloned()
+        });
+
+        let group_count = group_infos.len();
+        if group_count < 2 {
+            return;
+        }
+
+        let params = self.env.system_params_reader().await;
+        let barrier_interval_ms = params.barrier_interval_ms() as u64;
+        let checkpoint_secs = {
+            std::cmp::max(
+                1,
+                params.checkpoint_frequency() * barrier_interval_ms / 1000,
+            )
+        };
+
+        let mut left = 0;
+        let mut right = left + 1;
+
+        while left < right && right < group_count {
+            let group = &group_infos[left];
+            let next_group = &group_infos[right];
+            match self
+                .try_merge_compaction_group(
+                    &table_write_throughput,
+                    group,
+                    next_group,
+                    checkpoint_secs,
+                    &created_tables,
+                )
+                .await
+            {
+                Ok(_) => right += 1,
+                Err(e) => {
+                    tracing::debug!(
+                        error = %e.as_report(),
+                        "Failed to merge compaction group",
+                    );
+                    left = right;
+                    right = left + 1;
+                }
+            }
+        }
+    }
 }
diff --git a/src/meta/src/hummock/manager/transaction.rs b/src/meta/src/hummock/manager/transaction.rs
index a8d3645d29037..054ae657d594d 100644
--- a/src/meta/src/hummock/manager/transaction.rs
+++ b/src/meta/src/hummock/manager/transaction.rs
@@ -21,9 +21,7 @@ use risingwave_hummock_sdk::change_log::ChangeLogDelta;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_watermark::TableWatermarks;
-use risingwave_hummock_sdk::version::{
-    GroupDelta, HummockVersion, HummockVersionDelta, IntraLevelDelta,
-};
+use risingwave_hummock_sdk::version::{GroupDelta, HummockVersion, HummockVersionDelta};
 use risingwave_hummock_sdk::{CompactionGroupId, FrontendHummockVersionDelta, HummockVersionId};
 use risingwave_pb::hummock::{
     CompactionConfig, CompatibilityVersion, GroupConstruct, HummockVersionDeltas,
@@ -152,13 +150,7 @@ impl<'a> HummockVersionTransaction<'a> {
                 .entry(compaction_group_id)
                 .or_default()
                 .group_deltas;
-            let group_delta = GroupDelta::IntraLevel(IntraLevelDelta::new(
-                0,
-                0,      // l0_sub_level_id will be generated during apply_version_delta
-                vec![], // default
-                inserted_table_infos,
-                0, // default
-            ));
+            let group_delta = GroupDelta::NewL0SubLevel(inserted_table_infos);
 
             group_deltas.push(group_delta);
         }
@@ -207,7 +199,7 @@ impl<'a> HummockVersionTransaction<'a> {
     }
 }
 
-impl<'a> InMemValTransaction for HummockVersionTransaction<'a> {
+impl InMemValTransaction for HummockVersionTransaction<'_> {
     fn commit(self) {
         if let Some((version, deltas)) = self.pre_applied_version {
             *self.orig_version = version;
@@ -266,7 +258,7 @@ pub(super) struct SingleDeltaTransaction<'a, 'b> {
     delta: Option<HummockVersionDelta>,
 }
 
-impl<'a, 'b> SingleDeltaTransaction<'a, 'b> {
+impl SingleDeltaTransaction<'_, '_> {
     pub(super) fn latest_version(&self) -> &HummockVersion {
         self.version_txn.latest_version()
     }
@@ -286,7 +278,7 @@ impl<'a, 'b> SingleDeltaTransaction<'a, 'b> {
     }
 }
 
-impl<'a, 'b> Deref for SingleDeltaTransaction<'a, 'b> {
+impl Deref for SingleDeltaTransaction<'_, '_> {
     type Target = HummockVersionDelta;
 
     fn deref(&self) -> &Self::Target {
@@ -294,13 +286,13 @@ impl<'a, 'b> Deref for SingleDeltaTransaction<'a, 'b> {
     }
 }
 
-impl<'a, 'b> DerefMut for SingleDeltaTransaction<'a, 'b> {
+impl DerefMut for SingleDeltaTransaction<'_, '_> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         self.delta.as_mut().expect("should exist")
     }
 }
 
-impl<'a, 'b> Drop for SingleDeltaTransaction<'a, 'b> {
+impl Drop for SingleDeltaTransaction<'_, '_> {
     fn drop(&mut self) {
         if let Some(delta) = self.delta.take() {
             self.version_txn.pre_apply(delta);
@@ -325,7 +317,7 @@ impl<'a> HummockVersionStatsTransaction<'a> {
     }
 }
 
-impl<'a> InMemValTransaction for HummockVersionStatsTransaction<'a> {
+impl InMemValTransaction for HummockVersionStatsTransaction<'_> {
     fn commit(self) {
         if self.stats.has_new_value() {
             let stats = self.stats.clone();
@@ -345,7 +337,7 @@ where
     }
 }
 
-impl<'a> Deref for HummockVersionStatsTransaction<'a> {
+impl Deref for HummockVersionStatsTransaction<'_> {
     type Target = HummockVersionStats;
 
     fn deref(&self) -> &Self::Target {
@@ -353,7 +345,7 @@ impl<'a> Deref for HummockVersionStatsTransaction<'a> {
     }
 }
 
-impl<'a> DerefMut for HummockVersionStatsTransaction<'a> {
+impl DerefMut for HummockVersionStatsTransaction<'_> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         self.stats.deref_mut()
     }
diff --git a/src/meta/src/hummock/manager/versioning.rs b/src/meta/src/hummock/manager/versioning.rs
index 4d81689b9f705..fd516a2ba3e1e 100644
--- a/src/meta/src/hummock/manager/versioning.rs
+++ b/src/meta/src/hummock/manager/versioning.rs
@@ -36,7 +36,6 @@ use crate::hummock::error::Result;
 use crate::hummock::manager::checkpoint::HummockVersionCheckpoint;
 use crate::hummock::manager::commit_multi_var;
 use crate::hummock::manager::context::ContextInfo;
-use crate::hummock::manager::gc::DeleteObjectTracker;
 use crate::hummock::manager::transaction::HummockVersionTransaction;
 use crate::hummock::metrics_utils::{trigger_write_stop_stats, LocalTableMetrics};
 use crate::hummock::model::CompactionGroup;
@@ -56,8 +55,6 @@ pub struct Versioning {
     pub time_travel_snapshot_interval_counter: u64,
     /// Used to avoid the attempts to rewrite the same SST to meta store
     pub last_time_travel_snapshot_sst_ids: HashSet<HummockSstableId>,
-    /// Whether time travel is enabled during last commit epoch.
-    pub time_travel_toggle_check: bool,
 
     // Persistent states below
     pub hummock_version_deltas: BTreeMap<HummockVersionId, HummockVersionDelta>,
@@ -82,22 +79,6 @@ impl ContextInfo {
 }
 
 impl Versioning {
-    /// Marks all objects <= `min_pinned_version_id` for deletion.
-    pub(super) fn mark_objects_for_deletion(
-        &self,
-        context_info: &ContextInfo,
-        delete_object_tracker: &DeleteObjectTracker,
-    ) {
-        let min_pinned_version_id = context_info.min_pinned_version_id();
-        delete_object_tracker.add(
-            self.checkpoint
-                .stale_objects
-                .iter()
-                .filter(|(version_id, _)| **version_id <= min_pinned_version_id)
-                .flat_map(|(_, stale_objects)| stale_objects.id.iter().cloned()),
-        );
-    }
-
     pub(super) fn mark_next_time_travel_version_snapshot(&mut self) {
         self.time_travel_snapshot_interval_counter = u64::MAX;
     }
diff --git a/src/meta/src/hummock/metrics_utils.rs b/src/meta/src/hummock/metrics_utils.rs
index ad39e2228fba0..a5b6aea1270f5 100644
--- a/src/meta/src/hummock/metrics_utils.rs
+++ b/src/meta/src/hummock/metrics_utils.rs
@@ -353,6 +353,7 @@ pub fn remove_compaction_group_in_sst_stat(
     remove_compacting_task_stat(metrics, compaction_group_id, max_level);
     remove_split_stat(metrics, compaction_group_id);
     remove_compact_task_metrics(metrics, compaction_group_id, max_level);
+    remove_compaction_group_stat(metrics, compaction_group_id);
 }
 
 pub fn remove_compacting_task_stat(
@@ -388,6 +389,32 @@ pub fn remove_split_stat(metrics: &MetaMetrics, compaction_group_id: CompactionG
         .ok();
 }
 
+pub fn remove_compaction_group_stat(metrics: &MetaMetrics, compaction_group_id: CompactionGroupId) {
+    let label_str = compaction_group_id.to_string();
+    metrics
+        .compaction_group_size
+        .remove_label_values(&[&label_str])
+        .ok();
+    metrics
+        .compaction_group_file_count
+        .remove_label_values(&[&label_str])
+        .ok();
+    metrics
+        .compaction_group_throughput
+        .remove_label_values(&[&label_str])
+        .ok();
+
+    metrics
+        .split_compaction_group_count
+        .remove_label_values(&[&label_str])
+        .ok();
+
+    metrics
+        .merge_compaction_group_count
+        .remove_label_values(&[&label_str])
+        .ok();
+}
+
 pub fn trigger_pin_unpin_version_state(
     metrics: &MetaMetrics,
     pinned_versions: &BTreeMap<HummockContextId, HummockPinnedVersion>,
diff --git a/src/meta/src/hummock/mock_hummock_meta_client.rs b/src/meta/src/hummock/mock_hummock_meta_client.rs
index 805db163587a0..75bc3f121f43f 100644
--- a/src/meta/src/hummock/mock_hummock_meta_client.rs
+++ b/src/meta/src/hummock/mock_hummock_meta_client.rs
@@ -177,13 +177,12 @@ impl HummockMetaClient for MockHummockMetaClient {
         {
             vec![]
         } else {
-            vec![NewTableFragmentInfo::Normal {
-                mv_table_id: None,
-                internal_table_ids: commit_table_ids
+            vec![NewTableFragmentInfo {
+                table_ids: commit_table_ids
                     .iter()
                     .cloned()
                     .map(TableId::from)
-                    .collect_vec(),
+                    .collect(),
             }]
         };
 
diff --git a/src/meta/src/hummock/model/compaction_group_config.rs b/src/meta/src/hummock/model/compaction_group_config.rs
index ed7c3fc4916a7..7283e5faa72ae 100644
--- a/src/meta/src/hummock/model/compaction_group_config.rs
+++ b/src/meta/src/hummock/model/compaction_group_config.rs
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use risingwave_hummock_sdk::CompactionGroupId;
 use risingwave_pb::hummock::CompactionConfig;
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct CompactionGroup {
     pub group_id: CompactionGroupId,
     pub compaction_config: Arc<CompactionConfig>,
@@ -63,3 +63,23 @@ impl From<&CompactionGroup> for risingwave_pb::hummock::CompactionGroup {
         }
     }
 }
+
+impl CompactionGroup {
+    pub fn max_estimated_group_size(&self) -> u64 {
+        let max_level = self.compaction_config.max_level as usize;
+        let base_level_size = self.compaction_config.max_bytes_for_level_base;
+        let level_multiplier = self.compaction_config.max_bytes_for_level_multiplier;
+
+        fn size_for_levels(level_index: usize, base_size: u64, multiplier: u64) -> u64 {
+            if level_index == 0 {
+                base_size
+            } else {
+                base_size * multiplier.pow(level_index as u32)
+            }
+        }
+
+        (0..max_level)
+            .map(|level_index| size_for_levels(level_index, base_level_size, level_multiplier))
+            .sum()
+    }
+}
diff --git a/src/meta/src/hummock/test_utils.rs b/src/meta/src/hummock/test_utils.rs
index 006b31475461d..b6631a672e385 100644
--- a/src/meta/src/hummock/test_utils.rs
+++ b/src/meta/src/hummock/test_utils.rs
@@ -336,6 +336,7 @@ pub async fn setup_compute_env_with_metric(
         compactor_streams_change_tx,
     )
     .await;
+
     let fake_host_address = HostAddress {
         host: "127.0.0.1".to_string(),
         port,
diff --git a/src/meta/src/hummock/vacuum.rs b/src/meta/src/hummock/vacuum.rs
index dcb2d95d7ca1d..bc04aa53b6a87 100644
--- a/src/meta/src/hummock/vacuum.rs
+++ b/src/meta/src/hummock/vacuum.rs
@@ -35,7 +35,7 @@ pub type VacuumManagerRef = Arc<VacuumManager>;
 pub struct VacuumManager {
     env: MetaSrvEnv,
     hummock_manager: HummockManagerRef,
-    backup_manager: BackupManagerRef,
+    pub backup_manager: BackupManagerRef,
     /// Use the `CompactorManager` to dispatch `VacuumTask`.
     compactor_manager: CompactorManagerRef,
     /// SST object ids which have been dispatched to vacuum nodes but are not replied yet.
@@ -112,8 +112,7 @@ impl VacuumManager {
                 pending_object_ids
             } else {
                 // 2. If no pending SST objects, then fetch new ones.
-                let mut objects_to_delete = self.hummock_manager.get_objects_to_delete();
-                self.filter_out_pinned_ssts(&mut objects_to_delete).await?;
+                let objects_to_delete = self.hummock_manager.get_objects_to_delete();
                 if objects_to_delete.is_empty() {
                     return Ok(vec![]);
                 }
@@ -178,29 +177,6 @@ impl VacuumManager {
         Ok(sent_batch)
     }
 
-    async fn filter_out_pinned_ssts(
-        &self,
-        objects_to_delete: &mut Vec<HummockSstableObjectId>,
-    ) -> MetaResult<()> {
-        if objects_to_delete.is_empty() {
-            return Ok(());
-        }
-        let reject = self.backup_manager.list_pinned_ssts();
-        // Ack these SSTs immediately, because they tend to be pinned for long time.
-        // They will be GCed during full GC when they are no longer pinned.
-        let to_ack = objects_to_delete
-            .iter()
-            .filter(|s| reject.contains(s))
-            .cloned()
-            .collect_vec();
-        if to_ack.is_empty() {
-            return Ok(());
-        }
-        self.hummock_manager.ack_deleted_objects(&to_ack).await?;
-        objects_to_delete.retain(|s| !reject.contains(s));
-        Ok(())
-    }
-
     /// Acknowledges deletion of SSTs and deletes corresponding metadata.
     pub async fn report_vacuum_task(&self, vacuum_task: VacuumTask) -> MetaResult<()> {
         let deleted_object_ids = self
@@ -225,6 +201,7 @@ impl VacuumManager {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
     use std::sync::Arc;
 
     use itertools::Itertools;
@@ -275,7 +252,31 @@ mod tests {
             .await
             .unwrap();
         hummock_manager.create_version_checkpoint(0).await.unwrap();
-        assert!(!hummock_manager.get_objects_to_delete().is_empty());
+        assert!(hummock_manager.get_objects_to_delete().is_empty());
+        hummock_manager
+            .complete_full_gc(
+                sst_infos
+                    .iter()
+                    .flat_map(|ssts| ssts.iter().map(|s| s.object_id))
+                    .collect(),
+                None,
+                HashSet::default(),
+            )
+            .await
+            .unwrap();
+        assert_eq!(hummock_manager.get_objects_to_delete().len(), 3);
+        assert_eq!(
+            hummock_manager
+                .get_objects_to_delete()
+                .into_iter()
+                .sorted()
+                .collect::<Vec<_>>(),
+            sst_infos[0]
+                .iter()
+                .map(|s| s.object_id)
+                .sorted()
+                .collect::<Vec<_>>()
+        );
         // No SST deletion is scheduled because no available worker.
         assert_eq!(vacuum.vacuum_object().await.unwrap().len(), 0);
         let _receiver = compactor_manager.add_compactor(context_id);
diff --git a/src/meta/src/lib.rs b/src/meta/src/lib.rs
index 69d9dc21a075a..7b80eddb347d5 100644
--- a/src/meta/src/lib.rs
+++ b/src/meta/src/lib.rs
@@ -26,12 +26,10 @@
 #![cfg_attr(coverage, feature(coverage_attribute))]
 #![feature(custom_test_frameworks)]
 #![test_runner(risingwave_test_runner::test_runner::run_failpont_tests)]
-#![feature(is_sorted)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(const_option)]
 #![feature(anonymous_lifetime_in_impl_trait)]
 #![feature(duration_millis_float)]
-#![feature(option_get_or_insert_default)]
 
 pub mod backup_restore;
 pub mod barrier;
@@ -49,6 +47,7 @@ pub mod stream;
 pub mod telemetry;
 
 pub use error::{MetaError, MetaResult};
+use risingwave_common::config::MetaStoreConfig;
 pub use rpc::{ElectionClient, ElectionMember};
 
 use crate::manager::MetaOpts;
@@ -56,5 +55,8 @@ use crate::manager::MetaOpts;
 #[derive(Debug)]
 pub enum MetaStoreBackend {
     Mem,
-    Sql { endpoint: String },
+    Sql {
+        endpoint: String,
+        config: MetaStoreConfig,
+    },
 }
diff --git a/src/meta/src/manager/diagnose.rs b/src/meta/src/manager/diagnose.rs
index de8f983056a6a..f2d2cd58dd494 100644
--- a/src/meta/src/manager/diagnose.rs
+++ b/src/meta/src/manager/diagnose.rs
@@ -28,7 +28,7 @@ use risingwave_pb::meta::event_log::Event;
 use risingwave_pb::meta::EventLog;
 use risingwave_pb::monitor_service::StackTraceResponse;
 use risingwave_rpc_client::ComputeClientPool;
-use risingwave_sqlparser::ast::{CompatibleSourceSchema, Statement, Value};
+use risingwave_sqlparser::ast::{CompatibleFormatEncode, Statement, Value};
 use risingwave_sqlparser::parser::Parser;
 use serde_json::json;
 use thiserror_ext::AsReport;
@@ -731,28 +731,28 @@ fn redact_all_sql_options(sql: &str) -> Option<String> {
         let options = match statement {
             Statement::CreateTable {
                 with_options,
-                source_schema,
+                format_encode,
                 ..
             } => {
-                let connector_schema = match source_schema {
-                    Some(CompatibleSourceSchema::V2(cs)) => Some(&mut cs.row_options),
+                let format_encode = match format_encode {
+                    Some(CompatibleFormatEncode::V2(cs)) => Some(&mut cs.row_options),
                     _ => None,
                 };
-                (Some(with_options), connector_schema)
+                (Some(with_options), format_encode)
             }
             Statement::CreateSource { stmt } => {
-                let connector_schema = match &mut stmt.source_schema {
-                    CompatibleSourceSchema::V2(cs) => Some(&mut cs.row_options),
+                let format_encode = match &mut stmt.format_encode {
+                    CompatibleFormatEncode::V2(cs) => Some(&mut cs.row_options),
                     _ => None,
                 };
-                (Some(&mut stmt.with_properties.0), connector_schema)
+                (Some(&mut stmt.with_properties.0), format_encode)
             }
             Statement::CreateSink { stmt } => {
-                let connector_schema = match &mut stmt.sink_schema {
+                let format_encode = match &mut stmt.sink_schema {
                     Some(cs) => Some(&mut cs.row_options),
                     _ => None,
                 };
-                (Some(&mut stmt.with_properties.0), connector_schema)
+                (Some(&mut stmt.with_properties.0), format_encode)
             }
             _ => (None, None),
         };
diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs
index 3a7f75098373c..2d11556f6ef80 100644
--- a/src/meta/src/manager/env.rs
+++ b/src/meta/src/manager/env.rs
@@ -126,6 +126,8 @@ pub struct MetaOpts {
     pub full_gc_interval_sec: u64,
     /// Max number of object per full GC job can fetch.
     pub full_gc_object_limit: u64,
+    /// Duration in seconds to retain garbage collection history data.
+    pub gc_history_retention_time_sec: u64,
     /// Max number of inflight time travel query.
     pub max_inflight_time_travel_query: u64,
     /// Enable sanity check when SSTs are committed
@@ -167,13 +169,8 @@ pub struct MetaOpts {
     /// Schedule `tombstone_reclaim_compaction` for all compaction groups with this interval.
     pub periodic_tombstone_reclaim_compaction_interval_sec: u64,
 
-    /// Schedule `periodic_scheduling_compaction_group_interval_sec` for all compaction groups with this interval.
-    pub periodic_scheduling_compaction_group_interval_sec: u64,
-
-    /// The size limit to split a large compaction group.
-    pub split_group_size_limit: u64,
-    /// The size limit to move a state-table to other group.
-    pub min_table_split_size: u64,
+    /// Schedule `periodic_scheduling_compaction_group_split_interval_sec` for all compaction groups with this interval.
+    pub periodic_scheduling_compaction_group_split_interval_sec: u64,
 
     /// Whether config object storage bucket lifecycle to purge stale data.
     pub do_not_config_object_storage_lifecycle: bool,
@@ -181,9 +178,9 @@ pub struct MetaOpts {
     pub partition_vnode_count: u32,
 
     /// threshold of high write throughput of state-table, unit: B/sec
-    pub table_write_throughput_threshold: u64,
+    pub table_high_write_throughput_threshold: u64,
     /// threshold of low write throughput of state-table, unit: B/sec
-    pub min_table_split_write_throughput: u64,
+    pub table_low_write_throughput_threshold: u64,
 
     pub compaction_task_max_heartbeat_interval_secs: u64,
     pub compaction_task_max_progress_interval_secs: u64,
@@ -214,6 +211,23 @@ pub struct MetaOpts {
     /// l0 multi level picker whether to check the overlap accuracy between sub levels
     pub enable_check_task_level_overlap: bool,
     pub enable_dropped_column_reclaim: bool,
+
+    /// Whether to split the compaction group when the size of the group exceeds the threshold.
+    pub split_group_size_ratio: f64,
+
+    /// To split the compaction group when the high throughput statistics of the group exceeds the threshold.
+    pub table_stat_high_write_throughput_ratio_for_split: f64,
+
+    /// To merge the compaction group when the low throughput statistics of the group exceeds the threshold.
+    pub table_stat_low_write_throughput_ratio_for_merge: f64,
+
+    /// The window seconds of table throughput statistic history for split compaction group.
+    pub table_stat_throuput_window_seconds_for_split: usize,
+
+    /// The window seconds of table throughput statistic history for merge compaction group.
+    pub table_stat_throuput_window_seconds_for_merge: usize,
+
+    /// The configuration of the object store
     pub object_store_config: ObjectStoreConfig,
 
     /// The maximum number of trivial move tasks to be picked in a single loop
@@ -225,13 +239,13 @@ pub struct MetaOpts {
     pub compact_task_table_size_partition_threshold_low: u64,
     pub compact_task_table_size_partition_threshold_high: u64,
 
+    pub periodic_scheduling_compaction_group_merge_interval_sec: u64,
+
     // The private key for the secret store, used when the secret is stored in the meta.
     pub secret_store_private_key: Option<Vec<u8>>,
     /// The path of the temp secret file directory.
     pub temp_secret_file_dir: String,
 
-    pub table_info_statistic_history_times: usize,
-
     // Cluster limits
     pub actor_cnt_per_worker_parallelism_hard_limit: usize,
     pub actor_cnt_per_worker_parallelism_soft_limit: usize,
@@ -262,6 +276,7 @@ impl MetaOpts {
             min_sst_retention_time_sec: 3600 * 24 * 7,
             full_gc_interval_sec: 3600 * 24 * 7,
             full_gc_object_limit: 100_000,
+            gc_history_retention_time_sec: 3600 * 24 * 7,
             max_inflight_time_travel_query: 1000,
             enable_committed_sst_sanity_check: false,
             periodic_compaction_interval_sec: 60,
@@ -275,13 +290,11 @@ impl MetaOpts {
             telemetry_enabled: false,
             periodic_ttl_reclaim_compaction_interval_sec: 60,
             periodic_tombstone_reclaim_compaction_interval_sec: 60,
-            periodic_scheduling_compaction_group_interval_sec: 60,
-            split_group_size_limit: 5 * 1024 * 1024 * 1024,
-            min_table_split_size: 2 * 1024 * 1024 * 1024,
+            periodic_scheduling_compaction_group_split_interval_sec: 60,
             compact_task_table_size_partition_threshold_low: 128 * 1024 * 1024,
             compact_task_table_size_partition_threshold_high: 512 * 1024 * 1024,
-            table_write_throughput_threshold: 128 * 1024 * 1024,
-            min_table_split_write_throughput: 64 * 1024 * 1024,
+            table_high_write_throughput_threshold: 128 * 1024 * 1024,
+            table_low_write_throughput_threshold: 64 * 1024 * 1024,
             do_not_config_object_storage_lifecycle: true,
             partition_vnode_count: 32,
             compaction_task_max_heartbeat_interval_secs: 0,
@@ -299,11 +312,18 @@ impl MetaOpts {
             object_store_config: ObjectStoreConfig::default(),
             max_trivial_move_task_count_per_loop: 256,
             max_get_task_probe_times: 5,
-            secret_store_private_key: Some("0123456789abcdef".as_bytes().to_vec()),
+            secret_store_private_key: Some(
+                hex::decode("0123456789abcdef0123456789abcdef").unwrap(),
+            ),
             temp_secret_file_dir: "./secrets".to_string(),
-            table_info_statistic_history_times: 240,
             actor_cnt_per_worker_parallelism_hard_limit: usize::MAX,
             actor_cnt_per_worker_parallelism_soft_limit: usize::MAX,
+            split_group_size_ratio: 0.9,
+            table_stat_high_write_throughput_ratio_for_split: 0.5,
+            table_stat_low_write_throughput_ratio_for_merge: 0.7,
+            table_stat_throuput_window_seconds_for_split: 60,
+            table_stat_throuput_window_seconds_for_merge: 240,
+            periodic_scheduling_compaction_group_merge_interval_sec: 60 * 10,
             license_key_path: None,
         }
     }
diff --git a/src/meta/src/manager/event_log.rs b/src/meta/src/manager/event_log.rs
index a1fdc356502d9..f45437b6a7d03 100644
--- a/src/meta/src/manager/event_log.rs
+++ b/src/meta/src/manager/event_log.rs
@@ -172,6 +172,7 @@ impl From<&EventLog> for ChannelId {
             Event::CollectBarrierFail(_) => 6,
             Event::WorkerNodePanic(_) => 7,
             Event::AutoSchemaChangeFail(_) => 8,
+            Event::SinkFail(_) => 9,
         }
     }
 }
diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs
index 48dfbfaf55c0a..3e3b21929b470 100644
--- a/src/meta/src/manager/metadata.rs
+++ b/src/meta/src/manager/metadata.rs
@@ -13,13 +13,14 @@
 // limitations under the License.
 
 use std::collections::{BTreeMap, HashMap, HashSet};
+use std::fmt::Debug;
 use std::pin::pin;
 use std::time::Duration;
 
 use anyhow::anyhow;
 use futures::future::{select, Either};
 use risingwave_common::catalog::{TableId, TableOption};
-use risingwave_meta_model::{ObjectId, SourceId, WorkerId};
+use risingwave_meta_model::{DatabaseId, ObjectId, SourceId, WorkerId};
 use risingwave_pb::catalog::{PbSink, PbSource, PbTable};
 use risingwave_pb::common::worker_node::{PbResource, State};
 use risingwave_pb::common::{HostAddress, PbWorkerNode, PbWorkerType, WorkerNode, WorkerType};
@@ -58,13 +59,16 @@ pub(crate) enum ActiveStreamingWorkerChange {
 pub struct ActiveStreamingWorkerNodes {
     worker_nodes: HashMap<WorkerId, WorkerNode>,
     rx: UnboundedReceiver<LocalNotification>,
+    #[cfg_attr(not(debug_assertions), expect(dead_code))]
+    meta_manager: MetadataManager,
 }
 
 impl ActiveStreamingWorkerNodes {
-    pub(crate) fn uninitialized() -> Self {
+    pub(crate) fn uninitialized(meta_manager: MetadataManager) -> Self {
         Self {
             worker_nodes: Default::default(),
             rx: unbounded_channel().1,
+            meta_manager,
         }
     }
 
@@ -76,6 +80,7 @@ impl ActiveStreamingWorkerNodes {
         Ok(Self {
             worker_nodes: nodes.into_iter().map(|node| (node.id as _, node)).collect(),
             rx,
+            meta_manager,
         })
     }
 
@@ -191,6 +196,51 @@ impl ActiveStreamingWorkerNodes {
 
         ret
     }
+
+    #[cfg(debug_assertions)]
+    pub(crate) async fn validate_change(&self) {
+        use risingwave_pb::common::WorkerNode;
+        use thiserror_ext::AsReport;
+        match self
+            .meta_manager
+            .list_active_streaming_compute_nodes()
+            .await
+        {
+            Ok(worker_nodes) => {
+                let ignore_irrelevant_info = |node: &WorkerNode| {
+                    (
+                        node.id,
+                        WorkerNode {
+                            id: node.id,
+                            r#type: node.r#type,
+                            host: node.host.clone(),
+                            parallelism: node.parallelism,
+                            property: node.property.clone(),
+                            resource: node.resource.clone(),
+                            ..Default::default()
+                        },
+                    )
+                };
+                let worker_nodes: HashMap<_, _> =
+                    worker_nodes.iter().map(ignore_irrelevant_info).collect();
+                let curr_worker_nodes: HashMap<_, _> = self
+                    .current()
+                    .values()
+                    .map(ignore_irrelevant_info)
+                    .collect();
+                if worker_nodes != curr_worker_nodes {
+                    warn!(
+                        ?worker_nodes,
+                        ?curr_worker_nodes,
+                        "different to global snapshot"
+                    );
+                }
+            }
+            Err(e) => {
+                warn!(e = ?e.as_report(), "fail to list_active_streaming_compute_nodes to compare with local snapshot");
+            }
+        }
+    }
 }
 
 impl MetadataManager {
@@ -263,6 +313,47 @@ impl MetadataManager {
         self.cluster_controller.list_active_serving_workers().await
     }
 
+    pub async fn list_active_database_ids(&self) -> MetaResult<HashSet<DatabaseId>> {
+        Ok(self
+            .catalog_controller
+            .list_fragment_database_ids(None)
+            .await?
+            .into_iter()
+            .map(|(_, database_id)| database_id)
+            .collect())
+    }
+
+    pub async fn split_fragment_map_by_database<T: Debug>(
+        &self,
+        fragment_map: HashMap<FragmentId, T>,
+    ) -> MetaResult<HashMap<DatabaseId, HashMap<FragmentId, T>>> {
+        let fragment_to_database_map: HashMap<_, _> = self
+            .catalog_controller
+            .list_fragment_database_ids(Some(
+                fragment_map
+                    .keys()
+                    .map(|fragment_id| *fragment_id as _)
+                    .collect(),
+            ))
+            .await?
+            .into_iter()
+            .map(|(fragment_id, database_id)| {
+                (fragment_id as FragmentId, database_id as DatabaseId)
+            })
+            .collect();
+        let mut ret: HashMap<_, HashMap<_, _>> = HashMap::new();
+        for (fragment_id, value) in fragment_map {
+            let database_id = *fragment_to_database_map
+                .get(&fragment_id)
+                .ok_or_else(|| anyhow!("cannot get database_id of fragment {fragment_id}"))?;
+            ret.entry(database_id)
+                .or_default()
+                .try_insert(fragment_id, value)
+                .expect("non duplicate");
+        }
+        Ok(ret)
+    }
+
     pub async fn list_background_creating_jobs(&self) -> MetaResult<Vec<TableId>> {
         let tables = self
             .catalog_controller
@@ -532,7 +623,7 @@ impl MetadataManager {
 
     pub async fn count_streaming_job(&self) -> MetaResult<usize> {
         self.catalog_controller
-            .list_streaming_job_states()
+            .list_streaming_job_infos()
             .await
             .map(|x| x.len())
     }
diff --git a/src/meta/src/manager/streaming_job.rs b/src/meta/src/manager/streaming_job.rs
index 5257b2c1aa24a..9d5b135095fb1 100644
--- a/src/meta/src/manager/streaming_job.rs
+++ b/src/meta/src/manager/streaming_job.rs
@@ -328,8 +328,4 @@ impl StreamingJob {
             StreamingJob::MaterializedView(_) | StreamingJob::Index(_, _) => Ok(HashSet::new()),
         }
     }
-
-    pub fn is_source_job(&self) -> bool {
-        matches!(self, StreamingJob::Source(_))
-    }
 }
diff --git a/src/meta/src/model/mod.rs b/src/meta/src/model/mod.rs
index 829ebb5890258..10d7f08448ade 100644
--- a/src/meta/src/model/mod.rs
+++ b/src/meta/src/model/mod.rs
@@ -113,7 +113,7 @@ impl<'a, T> VarTransaction<'a, T> {
     }
 }
 
-impl<'a, T> Deref for VarTransaction<'a, T> {
+impl<T> Deref for VarTransaction<'_, T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -124,7 +124,7 @@ impl<'a, T> Deref for VarTransaction<'a, T> {
     }
 }
 
-impl<'a, T: Clone> DerefMut for VarTransaction<'a, T> {
+impl<T: Clone> DerefMut for VarTransaction<'_, T> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         if self.new_value.is_none() {
             self.new_value.replace(self.orig_value_ref.clone());
@@ -133,7 +133,7 @@ impl<'a, T: Clone> DerefMut for VarTransaction<'a, T> {
     }
 }
 
-impl<'a, T> InMemValTransaction for VarTransaction<'a, T>
+impl<T> InMemValTransaction for VarTransaction<'_, T>
 where
     T: PartialEq,
 {
@@ -202,7 +202,7 @@ impl<'a, K: Ord, V: Clone> BTreeMapTransactionValueGuard<'a, K, V> {
     }
 }
 
-impl<'a, K: Ord, V: Clone> Deref for BTreeMapTransactionValueGuard<'a, K, V> {
+impl<K: Ord, V: Clone> Deref for BTreeMapTransactionValueGuard<'_, K, V> {
     type Target = V;
 
     fn deref(&self) -> &Self::Target {
@@ -216,7 +216,7 @@ impl<'a, K: Ord, V: Clone> Deref for BTreeMapTransactionValueGuard<'a, K, V> {
     }
 }
 
-impl<'a, K: Ord, V: Clone> DerefMut for BTreeMapTransactionValueGuard<'a, K, V> {
+impl<K: Ord, V: Clone> DerefMut for BTreeMapTransactionValueGuard<'_, K, V> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         let is_occupied = matches!(
             self.staging_entry.as_ref().unwrap(),
@@ -478,7 +478,7 @@ impl<'a, K: Ord + Debug, V: Clone> BTreeMapEntryTransaction<'a, K, V> {
     }
 }
 
-impl<'a, K, V> Deref for BTreeMapEntryTransaction<'a, K, V> {
+impl<K, V> Deref for BTreeMapEntryTransaction<'_, K, V> {
     type Target = V;
 
     fn deref(&self) -> &Self::Target {
@@ -486,13 +486,13 @@ impl<'a, K, V> Deref for BTreeMapEntryTransaction<'a, K, V> {
     }
 }
 
-impl<'a, K, V> DerefMut for BTreeMapEntryTransaction<'a, K, V> {
+impl<K, V> DerefMut for BTreeMapEntryTransaction<'_, K, V> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.new_value
     }
 }
 
-impl<'a, K: Ord, V: PartialEq> InMemValTransaction for BTreeMapEntryTransaction<'a, K, V> {
+impl<K: Ord, V: PartialEq> InMemValTransaction for BTreeMapEntryTransaction<'_, K, V> {
     fn commit(self) {
         self.tree_ref.insert(self.key, self.new_value);
     }
diff --git a/src/meta/src/model/stream.rs b/src/meta/src/model/stream.rs
index 7eb921e0befa4..272b7e824b170 100644
--- a/src/meta/src/model/stream.rs
+++ b/src/meta/src/model/stream.rs
@@ -18,8 +18,10 @@ use std::ops::AddAssign;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::{VirtualNode, WorkerSlotId};
+use risingwave_common::util::stream_graph_visitor;
 use risingwave_connector::source::SplitImpl;
 use risingwave_meta_model::{SourceId, WorkerId};
+use risingwave_pb::catalog::Table;
 use risingwave_pb::common::PbActorLocation;
 use risingwave_pb::meta::table_fragments::actor_status::ActorState;
 use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, State};
@@ -30,9 +32,7 @@ use risingwave_pb::meta::table_parallelism::{
 use risingwave_pb::meta::{PbTableFragments, PbTableParallelism};
 use risingwave_pb::plan_common::PbExprContext;
 use risingwave_pb::stream_plan::stream_node::NodeBody;
-use risingwave_pb::stream_plan::{
-    FragmentTypeFlag, PbFragmentTypeFlag, PbStreamContext, StreamActor, StreamNode,
-};
+use risingwave_pb::stream_plan::{FragmentTypeFlag, PbStreamContext, StreamActor, StreamNode};
 
 use super::{ActorId, FragmentId};
 use crate::model::MetadataModelResult;
@@ -339,17 +339,6 @@ impl TableFragments {
             .flat_map(|fragment| fragment.actors.iter().map(|actor| actor.actor_id))
     }
 
-    /// Check if the fragment type mask is injectable.
-    pub fn is_injectable(fragment_type_mask: u32) -> bool {
-        (fragment_type_mask
-            & (PbFragmentTypeFlag::Source as u32
-                | PbFragmentTypeFlag::Now as u32
-                | PbFragmentTypeFlag::Values as u32
-                | PbFragmentTypeFlag::BarrierRecv as u32
-                | PbFragmentTypeFlag::SnapshotBackfillStreamScan as u32))
-            != 0
-    }
-
     /// Returns mview actor ids.
     pub fn mview_actor_ids(&self) -> Vec<ActorId> {
         Self::filter_actor_ids(self, |fragment_type_mask| {
@@ -359,7 +348,7 @@ impl TableFragments {
     }
 
     /// Returns actor ids that need to be tracked when creating MV.
-    pub fn tracking_progress_actor_ids(&self) -> Vec<ActorId> {
+    pub fn tracking_progress_actor_ids(&self) -> Vec<(ActorId, BackfillUpstreamType)> {
         let mut actor_ids = vec![];
         for fragment in self.fragments.values() {
             if fragment.fragment_type_mask & FragmentTypeFlag::CdcFilter as u32 != 0 {
@@ -373,7 +362,12 @@ impl TableFragments {
                     | FragmentTypeFlag::SourceScan as u32))
                 != 0
             {
-                actor_ids.extend(fragment.actors.iter().map(|actor| actor.actor_id));
+                actor_ids.extend(fragment.actors.iter().map(|actor| {
+                    (
+                        actor.actor_id,
+                        BackfillUpstreamType::from_fragment_type_mask(fragment.fragment_type_mask),
+                    )
+                }));
             }
         }
         actor_ids
@@ -564,6 +558,37 @@ impl TableFragments {
         }
     }
 
+    /// Retrieve the **complete** internal tables map of the whole graph.
+    ///
+    /// Compared to [`crate::stream::StreamFragmentGraph::incomplete_internal_tables`],
+    /// the table catalogs returned here are complete, with all fields filled.
+    pub fn internal_tables(&self) -> BTreeMap<u32, Table> {
+        self.collect_tables_inner(true)
+    }
+
+    /// `internal_tables()` with additional table in `Materialize` node.
+    pub fn all_tables(&self) -> BTreeMap<u32, Table> {
+        self.collect_tables_inner(false)
+    }
+
+    fn collect_tables_inner(&self, internal_tables_only: bool) -> BTreeMap<u32, Table> {
+        let mut tables = BTreeMap::new();
+        for fragment in self.fragments.values() {
+            stream_graph_visitor::visit_stream_node_tables_inner(
+                &mut fragment.actors[0].nodes.clone().unwrap(),
+                internal_tables_only,
+                true,
+                |table, _| {
+                    let table_id = table.id;
+                    tables
+                        .try_insert(table_id, table.clone())
+                        .unwrap_or_else(|_| panic!("duplicated table id `{}`", table_id));
+                },
+            );
+        }
+        tables
+    }
+
     /// Returns the internal table ids without the mview table.
     pub fn internal_table_ids(&self) -> Vec<u32> {
         self.fragments
@@ -592,3 +617,36 @@ impl TableFragments {
         self
     }
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum BackfillUpstreamType {
+    MView,
+    Values,
+    Source,
+}
+
+impl BackfillUpstreamType {
+    pub fn from_fragment_type_mask(mask: u32) -> Self {
+        let is_mview = (mask & FragmentTypeFlag::StreamScan as u32) != 0;
+        let is_values = (mask & FragmentTypeFlag::Values as u32) != 0;
+        let is_source = (mask & FragmentTypeFlag::SourceScan as u32) != 0;
+
+        // Note: in theory we can have multiple backfill executors in one fragment, but currently it's not possible.
+        // See <https://github.com/risingwavelabs/risingwave/issues/6236>.
+        debug_assert!(
+            is_mview as u8 + is_values as u8 + is_source as u8 == 1,
+            "a backfill fragment should either be mview, value or source, found {:?}",
+            mask
+        );
+
+        if is_mview {
+            BackfillUpstreamType::MView
+        } else if is_values {
+            BackfillUpstreamType::Values
+        } else if is_source {
+            BackfillUpstreamType::Source
+        } else {
+            unreachable!("invalid fragment type mask: {}", mask);
+        }
+    }
+}
diff --git a/src/meta/src/rpc/cloud_provider.rs b/src/meta/src/rpc/cloud_provider.rs
deleted file mode 100644
index fce20d5eea096..0000000000000
--- a/src/meta/src/rpc/cloud_provider.rs
+++ /dev/null
@@ -1,337 +0,0 @@
-// Copyright 2024 RisingWave Labs
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-use std::collections::HashMap;
-
-use anyhow::anyhow;
-use aws_config::retry::RetryConfig;
-use aws_sdk_ec2::error::ProvideErrorMetadata;
-use aws_sdk_ec2::types::{Filter, ResourceType, State, Tag, TagSpecification, VpcEndpointType};
-use itertools::Itertools;
-use risingwave_pb::catalog::connection::private_link_service::PrivateLinkProvider;
-use risingwave_pb::catalog::connection::PrivateLinkService;
-
-use crate::{MetaError, MetaResult};
-
-#[derive(Clone)]
-pub struct AwsEc2Client {
-    client: aws_sdk_ec2::Client,
-    /// `vpc_id`: The VPC of the running RisingWave instance
-    vpc_id: String,
-    security_group_id: String,
-}
-
-impl AwsEc2Client {
-    pub async fn new(vpc_id: &str, security_group_id: &str) -> Self {
-        let sdk_config = aws_config::from_env()
-            .retry_config(RetryConfig::standard().with_max_attempts(4))
-            .load()
-            .await;
-        let client = aws_sdk_ec2::Client::new(&sdk_config);
-
-        Self {
-            client,
-            vpc_id: vpc_id.to_string(),
-            security_group_id: security_group_id.to_string(),
-        }
-    }
-
-    pub async fn delete_vpc_endpoint(&self, vpc_endpoint_id: &str) -> MetaResult<()> {
-        let output = self
-            .client
-            .delete_vpc_endpoints()
-            .vpc_endpoint_ids(vpc_endpoint_id)
-            .send()
-            .await
-            .map_err(|e| {
-                anyhow!(
-                    "Failed to delete VPC endpoint. endpoint_id {vpc_endpoint_id}, error: {:?}, aws_request_id: {:?}",
-                    e.message(),
-                    e.meta().extra("aws_request_id")
-                )
-            })?;
-
-        if !output.unsuccessful().is_empty() {
-            return Err(MetaError::from(anyhow!(
-                "Failed to delete VPC endpoint {}, error: {:?}",
-                vpc_endpoint_id,
-                output.unsuccessful()
-            )));
-        }
-        Ok(())
-    }
-
-    /// `service_name`: The name of the endpoint service we want to access
-    /// `tags_user_str`: The tags specified in with clause of `create connection`
-    /// `tags_env`: The default tags specified in env var `RW_PRIVATELINK_ENDPOINT_DEFAULT_TAGS`
-    pub async fn create_aws_private_link(
-        &self,
-        service_name: &str,
-        tags_user_str: Option<&str>,
-        tags_env: Option<Vec<(&str, &str)>>,
-    ) -> MetaResult<PrivateLinkService> {
-        // fetch the AZs of the endpoint service
-        let service_azs = self.get_endpoint_service_az_names(service_name).await?;
-        let subnet_and_azs = self.describe_subnets(&self.vpc_id, &service_azs).await?;
-
-        let subnet_ids: Vec<String> = subnet_and_azs.iter().map(|(id, _, _)| id.clone()).collect();
-        let az_to_azid_map: HashMap<String, String> = subnet_and_azs
-            .into_iter()
-            .map(|(_, az, az_id)| (az, az_id))
-            .collect();
-
-        let tags_vec = match tags_user_str {
-            Some(tags_user_str) => {
-                let mut tags_user = tags_user_str
-                    .split(',')
-                    .map(|s| {
-                        s.split_once('=').ok_or_else(|| {
-                            MetaError::invalid_parameter("Failed to parse `tags` parameter")
-                        })
-                    })
-                    .collect::<MetaResult<Vec<(&str, &str)>>>()?;
-                match tags_env {
-                    Some(tags_env) => {
-                        tags_user.extend(tags_env);
-                        Some(tags_user)
-                    }
-                    None => Some(tags_user),
-                }
-            }
-            None => tags_env,
-        };
-
-        let (endpoint_id, endpoint_dns_names) = self
-            .create_vpc_endpoint(
-                &self.vpc_id,
-                service_name,
-                &self.security_group_id,
-                &subnet_ids,
-                tags_vec,
-            )
-            .await?;
-
-        // The number of returned DNS names may not equal to the input AZs,
-        // because some AZs may not have a subnet in the RW VPC
-        let mut azid_to_dns_map = HashMap::new();
-        if endpoint_dns_names.first().is_none() {
-            return Err(MetaError::from(anyhow!(
-                "No DNS name returned for the endpoint"
-            )));
-        }
-
-        // The first dns name doesn't has AZ info
-        let endpoint_dns_name = endpoint_dns_names.first().unwrap().clone();
-        for dns_name in &endpoint_dns_names {
-            for az in az_to_azid_map.keys() {
-                if dns_name.contains(az) {
-                    azid_to_dns_map
-                        .insert(az_to_azid_map.get(az).unwrap().clone(), dns_name.clone());
-                    break;
-                }
-            }
-        }
-
-        Ok(PrivateLinkService {
-            provider: PrivateLinkProvider::Aws.into(),
-            service_name: service_name.to_string(),
-            endpoint_id,
-            dns_entries: azid_to_dns_map,
-            endpoint_dns_name,
-        })
-    }
-
-    pub async fn is_vpc_endpoint_ready(&self, vpc_endpoint_id: &str) -> MetaResult<bool> {
-        let mut is_ready = false;
-        let filter = Filter::builder()
-            .name("vpc-endpoint-id")
-            .values(vpc_endpoint_id)
-            .build();
-        let output = self
-            .client
-            .describe_vpc_endpoints()
-            .set_filters(Some(vec![filter]))
-            .send()
-            .await
-            .map_err(|e| {
-                anyhow!(
-                    "Failed to check availability of VPC endpoint. endpoint_id: {vpc_endpoint_id}, error: {:?}, aws_request_id: {:?}",
-                    e.message(),
-                    e.meta().extra("aws_request_id")
-                )
-            })?;
-
-        match output.vpc_endpoints {
-            Some(endpoints) => {
-                let endpoint = endpoints
-                    .into_iter()
-                    .exactly_one()
-                    .map_err(|_| anyhow!("More than one VPC endpoint found with the same ID"))?;
-                if let Some(state) = endpoint.state {
-                    match state {
-                        State::Available => {
-                            is_ready = true;
-                        }
-                        // forward-compatible with protocol change
-                        other => {
-                            is_ready = other.as_str().eq_ignore_ascii_case("available");
-                        }
-                    }
-                }
-            }
-            None => {
-                return Err(MetaError::from(anyhow!(
-                    "No VPC endpoint found with the ID {}",
-                    vpc_endpoint_id
-                )));
-            }
-        }
-        Ok(is_ready)
-    }
-
-    async fn get_endpoint_service_az_names(&self, service_name: &str) -> MetaResult<Vec<String>> {
-        let mut service_azs = Vec::new();
-        let output = self
-            .client
-            .describe_vpc_endpoint_services()
-            .set_service_names(Some(vec![service_name.to_string()]))
-            .send()
-            .await
-            .map_err(|e| {
-                anyhow!(
-                    "Failed to describe VPC endpoint service, error: {:?}, aws_request_id: {:?}",
-                    e.message(),
-                    e.meta().extra("aws_request_id")
-                )
-            })?;
-
-        match output.service_details {
-            Some(details) => {
-                let detail = details.into_iter().exactly_one().map_err(|_| {
-                    anyhow!("More than one VPC endpoint service found with the same name")
-                })?;
-                if let Some(azs) = detail.availability_zones {
-                    service_azs.extend(azs.into_iter());
-                }
-            }
-            None => {
-                return Err(MetaError::from(anyhow!(
-                    "No VPC endpoint service found with the name {}",
-                    service_name
-                )));
-            }
-        }
-        Ok(service_azs)
-    }
-
-    async fn describe_subnets(
-        &self,
-        vpc_id: &str,
-        az_names: &[String],
-    ) -> MetaResult<Vec<(String, String, String)>> {
-        let vpc_filter = Filter::builder().name("vpc-id").values(vpc_id).build();
-        let az_filter = Filter::builder()
-            .name("availability-zone")
-            .set_values(Some(Vec::from(az_names)))
-            .build();
-        let output = self
-            .client
-            .describe_subnets()
-            .set_filters(Some(vec![vpc_filter, az_filter]))
-            .send()
-            .await
-            .map_err(|e| {
-                anyhow!("Failed to describe subnets for vpc_id {vpc_id}. error: {:?}, aws_request_id: {:?}",
-                    e.message(),
-                    e.meta().extra("aws_request_id"))
-            })?;
-
-        let subnets = output
-            .subnets
-            .unwrap_or_default()
-            .into_iter()
-            .unique_by(|s| s.availability_zone().unwrap_or_default().to_string())
-            .map(|s| {
-                (
-                    s.subnet_id.unwrap_or_default(),
-                    s.availability_zone.unwrap_or_default(),
-                    s.availability_zone_id.unwrap_or_default(),
-                )
-            })
-            .collect();
-        Ok(subnets)
-    }
-
-    async fn create_vpc_endpoint(
-        &self,
-        vpc_id: &str,
-        service_name: &str,
-        security_group_id: &str,
-        subnet_ids: &[String],
-        tags_vec: Option<Vec<(&str, &str)>>,
-    ) -> MetaResult<(String, Vec<String>)> {
-        let tag_spec = match tags_vec {
-            Some(tags_vec) => {
-                let tags = tags_vec
-                    .into_iter()
-                    .map(|(tag_key, tag_val)| {
-                        Tag::builder()
-                            .set_key(Some(tag_key.to_string()))
-                            .set_value(Some(tag_val.to_string()))
-                            .build()
-                    })
-                    .collect();
-                Some(vec![TagSpecification::builder()
-                    .set_resource_type(Some(ResourceType::VpcEndpoint))
-                    .set_tags(Some(tags))
-                    .build()])
-            }
-            None => None,
-        };
-
-        let output = self
-            .client
-            .create_vpc_endpoint()
-            .vpc_endpoint_type(VpcEndpointType::Interface)
-            .vpc_id(vpc_id)
-            .security_group_ids(security_group_id)
-            .service_name(service_name)
-            .set_subnet_ids(Some(subnet_ids.to_owned()))
-            .set_tag_specifications(tag_spec)
-            .send()
-            .await
-            .map_err(|e| {
-                anyhow!(
-                    "Failed to create vpc endpoint: vpc_id {vpc_id}, \
-                service_name {service_name}. error: {:?}, aws_request_id: {:?}",
-                    e.message(),
-                    e.meta().extra("aws_request_id")
-                )
-            })?;
-
-        let endpoint = output.vpc_endpoint().unwrap();
-        let mut dns_names = Vec::new();
-
-        endpoint.dns_entries().iter().for_each(|e| {
-            if let Some(dns_name) = e.dns_name() {
-                dns_names.push(dns_name.to_string());
-            }
-        });
-
-        Ok((
-            endpoint.vpc_endpoint_id().unwrap_or_default().to_string(),
-            dns_names,
-        ))
-    }
-}
diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs
index 995643215d317..fd336a1d0a5b4 100644
--- a/src/meta/src/rpc/ddl_controller.rs
+++ b/src/meta/src/rpc/ddl_controller.rs
@@ -20,7 +20,6 @@ use std::time::Duration;
 
 use anyhow::{anyhow, Context};
 use itertools::Itertools;
-use rand::Rng;
 use risingwave_common::bitmap::Bitmap;
 use risingwave_common::config::DefaultParallelism;
 use risingwave_common::hash::{ActorMapping, VnodeCountCompat};
@@ -28,14 +27,12 @@ use risingwave_common::secret::SecretEncryption;
 use risingwave_common::system_param::reader::SystemParamsRead;
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
 use risingwave_common::util::stream_graph_visitor::{
-    visit_fragment, visit_stream_node, visit_stream_node_cont_mut,
+    visit_stream_node, visit_stream_node_cont_mut,
 };
 use risingwave_common::{bail, hash, must_match};
 use risingwave_connector::error::ConnectorError;
-use risingwave_connector::source::cdc::CdcSourceType;
 use risingwave_connector::source::{
     ConnectorProperties, SourceEnumeratorContext, SourceProperties, SplitEnumerator,
-    UPSTREAM_SOURCE_KEY,
 };
 use risingwave_connector::{dispatch_source_prop, WithOptionsSecResolved};
 use risingwave_meta_model::object::ObjectType;
@@ -43,17 +40,14 @@ use risingwave_meta_model::{
     ConnectionId, DatabaseId, FunctionId, IndexId, ObjectId, SchemaId, SecretId, SinkId, SourceId,
     SubscriptionId, TableId, UserId, ViewId,
 };
-use risingwave_pb::catalog::connection::private_link_service::PbPrivateLinkProvider;
-use risingwave_pb::catalog::connection::PrivateLinkService;
-use risingwave_pb::catalog::source::OptionalAssociatedTableId;
-use risingwave_pb::catalog::table::OptionalAssociatedSourceId;
 use risingwave_pb::catalog::{
-    connection, Comment, Connection, CreateType, Database, Function, PbSink, PbSource, PbTable,
-    Schema, Secret, Sink, Source, Subscription, Table, View,
+    Comment, Connection, CreateType, Database, Function, PbSink, Schema, Secret, Sink, Source,
+    Subscription, Table, View,
 };
 use risingwave_pb::ddl_service::alter_owner_request::Object;
 use risingwave_pb::ddl_service::{
-    alter_name_request, alter_set_schema_request, DdlProgress, TableJobType, WaitVersion,
+    alter_name_request, alter_set_schema_request, alter_swap_rename_request, DdlProgress,
+    TableJobType, WaitVersion,
 };
 use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType;
 use risingwave_pb::meta::table_fragments::PbFragment;
@@ -67,7 +61,6 @@ use risingwave_pb::stream_plan::{
 use thiserror_ext::AsReport;
 use tokio::sync::Semaphore;
 use tokio::time::sleep;
-use tracing::log::warn;
 use tracing::Instrument;
 
 use crate::barrier::BarrierManagerRef;
@@ -79,7 +72,6 @@ use crate::manager::{
     IGNORED_NOTIFICATION_VERSION,
 };
 use crate::model::{FragmentId, StreamContext, TableFragments, TableParallelism};
-use crate::rpc::cloud_provider::AwsEc2Client;
 use crate::stream::{
     create_source_worker_handle, validate_sink, ActorGraphBuildResult, ActorGraphBuilder,
     CompleteStreamFragmentGraph, CreateStreamingJobContext, CreateStreamingJobOption,
@@ -148,6 +140,7 @@ pub enum DdlCommand {
     ),
     DropStreamingJob(StreamingJobId, DropMode, Option<ReplaceTableInfo>),
     AlterName(alter_name_request::Object, String),
+    AlterSwapRename(alter_swap_rename_request::Object),
     ReplaceTable(ReplaceTableInfo),
     AlterSourceColumn(Source),
     AlterObjectOwner(Object, UserId),
@@ -188,7 +181,6 @@ pub struct DdlController {
     pub(crate) source_manager: SourceManagerRef,
     barrier_manager: BarrierManagerRef,
 
-    aws_client: Arc<Option<AwsEc2Client>>,
     // The semaphore is used to limit the number of concurrent streaming job creation.
     pub(crate) creating_streaming_job_permits: Arc<CreatingStreamingJobPermit>,
 }
@@ -258,7 +250,6 @@ impl DdlController {
         stream_manager: GlobalStreamManagerRef,
         source_manager: SourceManagerRef,
         barrier_manager: BarrierManagerRef,
-        aws_client: Arc<Option<AwsEc2Client>>,
     ) -> Self {
         let creating_streaming_job_permits = Arc::new(CreatingStreamingJobPermit::new(&env).await);
         Self {
@@ -267,7 +258,6 @@ impl DdlController {
             stream_manager,
             source_manager,
             barrier_manager,
-            aws_client,
             creating_streaming_job_permits,
         }
     }
@@ -349,18 +339,14 @@ impl DdlController {
                 DdlCommand::DropSubscription(subscription_id, drop_mode) => {
                     ctrl.drop_subscription(subscription_id, drop_mode).await
                 }
+                DdlCommand::AlterSwapRename(objects) => ctrl.alter_swap_rename(objects).await,
             }
         }
         .in_current_span();
         let notification_version = tokio::spawn(fut).await.map_err(|e| anyhow!(e))??;
         Ok(Some(WaitVersion {
             catalog_version: notification_version,
-            hummock_version_id: self
-                .barrier_manager
-                .hummock_manager()
-                .get_version_id()
-                .await
-                .to_u64(),
+            hummock_version_id: self.barrier_manager.get_hummock_version_id().await.to_u64(),
         }))
     }
 
@@ -551,21 +537,6 @@ impl DdlController {
             .await
     }
 
-    pub(crate) async fn delete_vpc_endpoint(&self, svc: &PrivateLinkService) -> MetaResult<()> {
-        // delete AWS vpc endpoint
-        if svc.get_provider()? == PbPrivateLinkProvider::Aws {
-            if let Some(aws_cli) = self.aws_client.as_ref() {
-                aws_cli.delete_vpc_endpoint(&svc.endpoint_id).await?;
-            } else {
-                warn!(
-                    "AWS client is not initialized, skip deleting vpc endpoint {}",
-                    svc.endpoint_id
-                );
-            }
-        }
-        Ok(())
-    }
-
     async fn create_subscription(
         &self,
         mut subscription: Subscription,
@@ -605,19 +576,20 @@ impl DdlController {
     ) -> MetaResult<NotificationVersion> {
         tracing::debug!("preparing drop subscription");
         let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await;
-        let table_id = self
+        let subscription = self
             .metadata_manager
             .catalog_controller
             .get_subscription_by_id(subscription_id)
-            .await?
-            .dependent_table_id;
+            .await?;
+        let table_id = subscription.dependent_table_id;
+        let database_id = subscription.database_id.into();
         let (_, version) = self
             .metadata_manager
             .catalog_controller
             .drop_relation(ObjectType::Subscription, subscription_id as _, drop_mode)
             .await?;
         self.stream_manager
-            .drop_subscription(subscription_id as _, table_id)
+            .drop_subscription(database_id, subscription_id as _, table_id)
             .await;
         tracing::debug!("finish drop subscription");
         Ok(version)
@@ -932,7 +904,7 @@ impl DdlController {
     pub async fn create_streaming_job(
         &self,
         mut streaming_job: StreamingJob,
-        mut fragment_graph: StreamFragmentGraphProto,
+        fragment_graph: StreamFragmentGraphProto,
         affected_table_replace_info: Option<ReplaceTableInfo>,
     ) -> MetaResult<NotificationVersion> {
         let ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap());
@@ -947,24 +919,6 @@ impl DdlController {
             .await?;
         let job_id = streaming_job.id();
 
-        match &mut streaming_job {
-            StreamingJob::Table(src, table, job_type) => {
-                // If we're creating a table with connector, we should additionally fill its ID first.
-                fill_table_stream_graph_info(src, table, *job_type, &mut fragment_graph);
-            }
-            StreamingJob::Source(src) => {
-                // set the inner source id of source node.
-                for fragment in fragment_graph.fragments.values_mut() {
-                    visit_fragment(fragment, |node_body| {
-                        if let NodeBody::Source(source_node) = node_body {
-                            source_node.source_inner.as_mut().unwrap().source_id = src.id;
-                        }
-                    });
-                }
-            }
-            _ => {}
-        }
-
         tracing::debug!(
             id = job_id,
             definition = streaming_job.definition(),
@@ -1009,7 +963,7 @@ impl DdlController {
                 self.env.event_log_manager_ref().add_event_logs(vec![
                     risingwave_pb::meta::event_log::Event::CreateStreamJobFail(event),
                 ]);
-                let aborted = self
+                let (aborted, _) = self
                     .metadata_manager
                     .catalog_controller
                     .try_abort_creating_streaming_job(job_id as _, false)
@@ -1040,11 +994,14 @@ impl DdlController {
         streaming_job.set_dml_fragment_id(fragment_graph.dml_fragment_id());
 
         // create internal table catalogs and refill table id.
-        let internal_tables = fragment_graph.internal_tables().into_values().collect_vec();
+        let incomplete_internal_tables = fragment_graph
+            .incomplete_internal_tables()
+            .into_values()
+            .collect_vec();
         let table_id_map = self
             .metadata_manager
             .catalog_controller
-            .create_internal_table_catalog(&streaming_job, internal_tables)
+            .create_internal_table_catalog(&streaming_job, incomplete_internal_tables)
             .await?;
         fragment_graph.refill_internal_table_ids(table_id_map);
 
@@ -1056,6 +1013,16 @@ impl DdlController {
                     ..
                 } = replace_table_info;
 
+                // Ensure the max parallelism unchanged before replacing table.
+                let original_max_parallelism = self
+                    .metadata_manager
+                    .get_job_max_parallelism(streaming_job.id().into())
+                    .await?;
+                let fragment_graph = PbStreamFragmentGraph {
+                    max_parallelism: original_max_parallelism as _,
+                    ..fragment_graph
+                };
+
                 let fragment_graph =
                     StreamFragmentGraph::new(&self.env, fragment_graph, &streaming_job)?;
                 streaming_job.set_table_fragment_id(fragment_graph.table_fragment_id());
@@ -1101,7 +1068,7 @@ impl DdlController {
 
         self.metadata_manager
             .catalog_controller
-            .prepare_streaming_job(table_fragments.to_protobuf(), streaming_job, false)
+            .prepare_streaming_job(&table_fragments, streaming_job, false)
             .await?;
 
         // create streaming jobs.
@@ -1161,14 +1128,11 @@ impl DdlController {
                     .await;
             }
             ObjectType::Connection => {
-                let (version, conn) = self
+                let (version, _conn) = self
                     .metadata_manager
                     .catalog_controller
                     .drop_connection(object_id)
                     .await?;
-                if let Some(connection::Info::PrivateLinkService(svc)) = &conn.info {
-                    self.delete_vpc_endpoint(svc).await?;
-                }
                 return Ok(version);
             }
             _ => {
@@ -1195,6 +1159,16 @@ impl DdlController {
                 panic!("additional replace table event only occurs when dropping sink into table")
             };
 
+            // Ensure the max parallelism unchanged before replacing table.
+            let original_max_parallelism = self
+                .metadata_manager
+                .get_job_max_parallelism(streaming_job.id().into())
+                .await?;
+            let fragment_graph = PbStreamFragmentGraph {
+                max_parallelism: original_max_parallelism as _,
+                ..fragment_graph
+            };
+
             let fragment_graph =
                 StreamFragmentGraph::new(&self.env, fragment_graph, &streaming_job)?;
             streaming_job.set_table_fragment_id(fragment_graph.table_fragment_id());
@@ -1234,7 +1208,7 @@ impl DdlController {
 
                 self.metadata_manager
                     .catalog_controller
-                    .prepare_streaming_job(table_fragments.to_protobuf(), &streaming_job, true)
+                    .prepare_streaming_job(&table_fragments, &streaming_job, true)
                     .await?;
 
                 self.stream_manager
@@ -1276,25 +1250,16 @@ impl DdlController {
         }
 
         let ReleaseContext {
+            database_id,
             streaming_job_ids,
             state_table_ids,
             source_ids,
-            connections,
             source_fragments,
             removed_actors,
             removed_fragments,
+            ..
         } = release_ctx;
 
-        // delete vpc endpoints.
-        for conn in connections {
-            let _ = self
-                .delete_vpc_endpoint(&conn.to_protobuf())
-                .await
-                .inspect_err(|err| {
-                    tracing::warn!(err = ?err.as_report(), "failed to delete vpc endpoint");
-                });
-        }
-
         // unregister sources.
         self.source_manager
             .unregister_sources(source_ids.into_iter().map(|id| id as _).collect())
@@ -1319,6 +1284,7 @@ impl DdlController {
         // drop streaming jobs.
         self.stream_manager
             .drop_streaming_jobs(
+                risingwave_common::catalog::DatabaseId::new(database_id as _),
                 removed_actors.into_iter().map(|id| id as _).collect(),
                 streaming_job_ids,
                 state_table_ids,
@@ -1341,6 +1307,16 @@ impl DdlController {
         let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await;
         let ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap());
 
+        // Ensure the max parallelism unchanged before replacing table.
+        let original_max_parallelism = self
+            .metadata_manager
+            .get_job_max_parallelism(streaming_job.id().into())
+            .await?;
+        let fragment_graph = PbStreamFragmentGraph {
+            max_parallelism: original_max_parallelism as _,
+            ..fragment_graph
+        };
+
         // 1. build fragment graph.
         let fragment_graph = StreamFragmentGraph::new(&self.env, fragment_graph, &streaming_job)?;
         streaming_job.set_table_fragment_id(fragment_graph.table_fragment_id());
@@ -1432,7 +1408,7 @@ impl DdlController {
 
             self.metadata_manager
                 .catalog_controller
-                .prepare_streaming_job(table_fragments.to_protobuf(), &streaming_job, true)
+                .prepare_streaming_job(&table_fragments, &streaming_job, true)
                 .await?;
 
             self.stream_manager
@@ -1560,7 +1536,6 @@ impl DdlController {
     ) -> MetaResult<(CreateStreamingJobContext, TableFragments)> {
         let id = stream_job.id();
         let specified_parallelism = fragment_graph.specified_parallelism();
-        let internal_tables = fragment_graph.internal_tables();
         let expr_context = stream_ctx.to_expr_context();
         let max_parallelism = NonZeroUsize::new(fragment_graph.max_parallelism()).unwrap();
 
@@ -1643,6 +1618,7 @@ impl DdlController {
             table_parallelism,
             max_parallelism.get(),
         );
+        let internal_tables = table_fragments.internal_tables();
 
         if let Some(mview_fragment) = table_fragments.mview_fragment() {
             stream_job.set_table_vnode_count(mview_fragment.vnode_count());
@@ -1885,6 +1861,55 @@ impl DdlController {
             .await
     }
 
+    async fn alter_swap_rename(
+        &self,
+        object: alter_swap_rename_request::Object,
+    ) -> MetaResult<NotificationVersion> {
+        let (obj_type, src_id, dst_id) = match object {
+            alter_swap_rename_request::Object::Schema(_) => unimplemented!("schema swap"),
+            alter_swap_rename_request::Object::Table(objs) => {
+                let (src_id, dst_id) = (
+                    objs.src_object_id as ObjectId,
+                    objs.dst_object_id as ObjectId,
+                );
+                (ObjectType::Table, src_id, dst_id)
+            }
+            alter_swap_rename_request::Object::View(objs) => {
+                let (src_id, dst_id) = (
+                    objs.src_object_id as ObjectId,
+                    objs.dst_object_id as ObjectId,
+                );
+                (ObjectType::View, src_id, dst_id)
+            }
+            alter_swap_rename_request::Object::Source(objs) => {
+                let (src_id, dst_id) = (
+                    objs.src_object_id as ObjectId,
+                    objs.dst_object_id as ObjectId,
+                );
+                (ObjectType::Source, src_id, dst_id)
+            }
+            alter_swap_rename_request::Object::Sink(objs) => {
+                let (src_id, dst_id) = (
+                    objs.src_object_id as ObjectId,
+                    objs.dst_object_id as ObjectId,
+                );
+                (ObjectType::Sink, src_id, dst_id)
+            }
+            alter_swap_rename_request::Object::Subscription(objs) => {
+                let (src_id, dst_id) = (
+                    objs.src_object_id as ObjectId,
+                    objs.dst_object_id as ObjectId,
+                );
+                (ObjectType::Subscription, src_id, dst_id)
+            }
+        };
+
+        self.metadata_manager
+            .catalog_controller
+            .alter_swap_rename(obj_type, src_id, dst_id)
+            .await
+    }
+
     async fn alter_owner(
         &self,
         object: Object,
@@ -1958,70 +1983,3 @@ impl DdlController {
             .await
     }
 }
-
-/// Fill in necessary information for `Table` stream graph.
-/// e.g., fill source id for table with connector, fill external table id for CDC table.
-pub fn fill_table_stream_graph_info(
-    source: &mut Option<PbSource>,
-    table: &mut PbTable,
-    table_job_type: TableJobType,
-    fragment_graph: &mut PbStreamFragmentGraph,
-) {
-    let mut source_count = 0;
-    for fragment in fragment_graph.fragments.values_mut() {
-        visit_fragment(fragment, |node_body| {
-            if let NodeBody::Source(source_node) = node_body {
-                if source_node.source_inner.is_none() {
-                    // skip empty source for dml node
-                    return;
-                }
-
-                // If we're creating a table with connector, we should additionally fill its ID first.
-                if let Some(source) = source {
-                    source_node.source_inner.as_mut().unwrap().source_id = source.id;
-                    source_count += 1;
-
-                    // Generate a random server id for mysql cdc source if needed
-                    // `server.id` (in the range from 1 to 2^32 - 1). This value MUST be unique across whole replication
-                    // group (that is, different from any other server id being used by any master or slave)
-                    if let Some(connector) = source.with_properties.get(UPSTREAM_SOURCE_KEY)
-                        && matches!(
-                            CdcSourceType::from(connector.as_str()),
-                            CdcSourceType::Mysql
-                        )
-                    {
-                        let props = &mut source_node.source_inner.as_mut().unwrap().with_properties;
-                        let rand_server_id = rand::thread_rng().gen_range(1..u32::MAX);
-                        props
-                            .entry("server.id".to_string())
-                            .or_insert(rand_server_id.to_string());
-
-                        // make these two `Source` consistent
-                        props.clone_into(&mut source.with_properties);
-                    }
-
-                    assert_eq!(
-                        source_count, 1,
-                        "require exactly 1 external stream source when creating table with a connector"
-                    );
-
-                    // Fill in the correct table id for source.
-                    source.optional_associated_table_id =
-                        Some(OptionalAssociatedTableId::AssociatedTableId(table.id));
-                    // Fill in the correct source id for mview.
-                    table.optional_associated_source_id =
-                        Some(OptionalAssociatedSourceId::AssociatedSourceId(source.id));
-                }
-            }
-
-            // fill table id for cdc backfill
-            if let NodeBody::StreamCdcScan(node) = node_body
-                && table_job_type == TableJobType::SharedCdcSource
-            {
-                if let Some(table_desc) = node.cdc_table_desc.as_mut() {
-                    table_desc.table_id = table.id;
-                }
-            }
-        });
-    }
-}
diff --git a/src/meta/src/rpc/metrics.rs b/src/meta/src/rpc/metrics.rs
index 2d5020edb04be..a30d2918d0c57 100644
--- a/src/meta/src/rpc/metrics.rs
+++ b/src/meta/src/rpc/metrics.rs
@@ -83,9 +83,6 @@ pub struct MetaMetrics {
     pub snapshot_backfill_barrier_latency: LabelGuardedHistogramVec<2>, // (table_id, barrier_type)
     /// The latency of commit epoch of `table_id`
     pub snapshot_backfill_wait_commit_latency: LabelGuardedHistogramVec<1>, // (table_id, )
-    /// The latency that the upstream waits on the snapshot backfill progress after the upstream
-    /// has collected the barrier.
-    pub snapshot_backfill_upstream_wait_progress_latency: LabelGuardedHistogramVec<1>, /* (table_id, ) */
     /// The lags between the upstream epoch and the downstream epoch.
     pub snapshot_backfill_lag: LabelGuardedIntGaugeVec<1>, // (table_id, )
     /// The number of inflight barriers of `table_id`
@@ -207,6 +204,11 @@ pub struct MetaMetrics {
     pub auto_schema_change_latency: LabelGuardedHistogramVec<2>,
 
     pub time_travel_version_replay_latency: Histogram,
+
+    pub compaction_group_count: IntGauge,
+    pub compaction_group_size: IntGaugeVec,
+    pub compaction_group_file_count: IntGaugeVec,
+    pub compaction_group_throughput: IntGaugeVec,
 }
 
 pub static GLOBAL_META_METRICS: LazyLock<MetaMetrics> =
@@ -282,13 +284,7 @@ impl MetaMetrics {
         );
         let snapshot_backfill_wait_commit_latency =
             register_guarded_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
-        let opts = histogram_opts!(
-            "meta_snapshot_backfill_upstream_wait_progress_latency",
-            "snapshot backfill upstream_wait_progress_latency",
-            exponential_buckets(0.1, 1.5, 20).unwrap() // max 221s
-        );
-        let snapshot_backfill_upstream_wait_progress_latency =
-            register_guarded_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap();
+
         let snapshot_backfill_lag = register_guarded_int_gauge_vec_with_registry!(
             "meta_snapshot_backfill_upstream_lag",
             "snapshot backfill upstream_lag",
@@ -749,6 +745,37 @@ impl MetaMetrics {
         let time_travel_version_replay_latency =
             register_histogram_with_registry!(opts, registry).unwrap();
 
+        let compaction_group_count = register_int_gauge_with_registry!(
+            "storage_compaction_group_count",
+            "The number of compaction groups",
+            registry,
+        )
+        .unwrap();
+
+        let compaction_group_size = register_int_gauge_vec_with_registry!(
+            "storage_compaction_group_size",
+            "The size of compaction group",
+            &["group"],
+            registry
+        )
+        .unwrap();
+
+        let compaction_group_file_count = register_int_gauge_vec_with_registry!(
+            "storage_compaction_group_file_count",
+            "The file count of compaction group",
+            &["group"],
+            registry
+        )
+        .unwrap();
+
+        let compaction_group_throughput = register_int_gauge_vec_with_registry!(
+            "storage_compaction_group_throughput",
+            "The throughput of compaction group",
+            &["group"],
+            registry
+        )
+        .unwrap();
+
         Self {
             grpc_latency,
             barrier_latency,
@@ -759,7 +786,6 @@ impl MetaMetrics {
             last_committed_barrier_time,
             snapshot_backfill_barrier_latency,
             snapshot_backfill_wait_commit_latency,
-            snapshot_backfill_upstream_wait_progress_latency,
             snapshot_backfill_lag,
             snapshot_backfill_inflight_barrier_num,
             recovery_failure_cnt,
@@ -824,6 +850,10 @@ impl MetaMetrics {
             auto_schema_change_latency,
             merge_compaction_group_count,
             time_travel_version_replay_latency,
+            compaction_group_count,
+            compaction_group_size,
+            compaction_group_file_count,
+            compaction_group_throughput,
         }
     }
 
diff --git a/src/meta/src/rpc/mod.rs b/src/meta/src/rpc/mod.rs
index 8b256d1b2145e..9f840ded5aa47 100644
--- a/src/meta/src/rpc/mod.rs
+++ b/src/meta/src/rpc/mod.rs
@@ -12,7 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-pub mod cloud_provider;
 pub mod ddl_controller;
 pub mod election;
 pub mod intercept;
diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs
index 2dbd6364d4e2c..4c866c16fb0f6 100644
--- a/src/meta/src/stream/scale.rs
+++ b/src/meta/src/stream/scale.rs
@@ -27,7 +27,7 @@ use num_integer::Integer;
 use num_traits::abs;
 use risingwave_common::bail;
 use risingwave_common::bitmap::{Bitmap, BitmapBuilder};
-use risingwave_common::catalog::TableId;
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_common::hash::ActorMapping;
 use risingwave_common::util::iter_util::ZipEqDebug;
 use risingwave_meta_model::{actor, fragment, ObjectId, StreamingParallelism, WorkerId};
@@ -52,7 +52,7 @@ use tokio::time::{Instant, MissedTickBehavior};
 use crate::barrier::{Command, Reschedule};
 use crate::controller::scale::RescheduleWorkingSet;
 use crate::manager::{LocalNotification, MetaSrvEnv, MetadataManager};
-use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments, TableParallelism};
+use crate::model::{ActorId, DispatcherId, FragmentId, TableParallelism};
 use crate::serving::{
     to_deleted_fragment_worker_slot_mapping, to_fragment_worker_slot_mapping, ServingVnodeMapping,
 };
@@ -850,7 +850,7 @@ impl ScaleController {
     /// - `reschedule_fragment`: the generated reschedule plan
     /// - `applied_reschedules`: the changes that need to be updated to the meta store (`pre_apply_reschedules`, only for V1).
     ///
-    /// In [normal process of scaling](`GlobalStreamManager::reschedule_actors_impl`), we use the returned values to
+    /// In [normal process of scaling](`GlobalStreamManager::reschedule_actors`), we use the returned values to
     /// build a [`Command::RescheduleFragment`], which will then flows through the barrier mechanism to perform scaling.
     /// Meta store is updated after the barrier is collected.
     ///
@@ -1291,12 +1291,6 @@ impl ScaleController {
 
         for (fragment_id, _) in reschedules {
             let mut actors_to_create: HashMap<_, Vec<_>> = HashMap::new();
-            let fragment_type_mask = ctx
-                .fragment_map
-                .get(&fragment_id)
-                .unwrap()
-                .fragment_type_mask;
-            let injectable = TableFragments::is_injectable(fragment_type_mask);
 
             if let Some(actor_worker_maps) = fragment_actors_to_create.get(&fragment_id).cloned() {
                 for (actor_id, worker_id) in actor_worker_maps {
@@ -1431,7 +1425,6 @@ impl ScaleController {
                     upstream_dispatcher_mapping,
                     downstream_fragment_ids,
                     actor_splits,
-                    injectable,
                     newly_created_actors: vec![],
                 },
             );
@@ -2312,16 +2305,7 @@ impl GlobalStreamManager {
     ///     * automatic parallelism control for [`TableParallelism::Adaptive`] when worker nodes changed
     pub async fn reschedule_actors(
         &self,
-        reschedules: HashMap<FragmentId, WorkerReschedule>,
-        options: RescheduleOptions,
-        table_parallelism: Option<HashMap<TableId, TableParallelism>>,
-    ) -> MetaResult<()> {
-        self.reschedule_actors_impl(reschedules, options, table_parallelism)
-            .await
-    }
-
-    async fn reschedule_actors_impl(
-        &self,
+        database_id: DatabaseId,
         reschedules: HashMap<FragmentId, WorkerReschedule>,
         options: RescheduleOptions,
         table_parallelism: Option<HashMap<TableId, TableParallelism>>,
@@ -2368,7 +2352,7 @@ impl GlobalStreamManager {
         let _source_pause_guard = self.source_manager.paused.lock().await;
 
         self.barrier_scheduler
-            .run_config_change_command_with_pause(command)
+            .run_config_change_command_with_pause(database_id, command)
             .await?;
 
         tracing::info!("reschedule done");
@@ -2497,15 +2481,22 @@ impl GlobalStreamManager {
             return Ok(false);
         };
 
-        self.reschedule_actors(
-            reschedules,
-            RescheduleOptions {
-                resolve_no_shuffle_upstream: false,
-                skip_create_new_actors: false,
-            },
-            None,
-        )
-        .await?;
+        for (database_id, reschedules) in self
+            .metadata_manager
+            .split_fragment_map_by_database(reschedules)
+            .await?
+        {
+            self.reschedule_actors(
+                DatabaseId::new(database_id as _),
+                reschedules,
+                RescheduleOptions {
+                    resolve_no_shuffle_upstream: false,
+                    skip_create_new_actors: false,
+                },
+                None,
+            )
+            .await?;
+        }
 
         Ok(true)
     }
diff --git a/src/meta/src/stream/source_manager.rs b/src/meta/src/stream/source_manager.rs
index c5bcc0c179ba3..ef8c1298264b9 100644
--- a/src/meta/src/stream/source_manager.rs
+++ b/src/meta/src/stream/source_manager.rs
@@ -29,7 +29,7 @@ use risingwave_connector::source::{
     SplitEnumerator, SplitId, SplitImpl, SplitMetaData,
 };
 use risingwave_connector::{dispatch_source_prop, WithOptionsSecResolved};
-use risingwave_meta_model::SourceId;
+use risingwave_meta_model::{DatabaseId, SourceId};
 use risingwave_pb::catalog::Source;
 use risingwave_pb::source::{ConnectorSplit, ConnectorSplits};
 use risingwave_pb::stream_plan::Dispatcher;
@@ -138,7 +138,6 @@ pub async fn create_source_worker_handle(
 
         tokio::spawn(async move { worker.run(sync_call_rx).await })
     });
-
     Ok(ConnectorSourceWorkerHandle {
         handle,
         sync_call_tx,
@@ -315,7 +314,7 @@ impl SourceManagerCore {
     ///
     /// `self.actor_splits` will not be updated. It will be updated by `Self::apply_source_change`,
     /// after the mutation barrier has been collected.
-    async fn reassign_splits(&self) -> MetaResult<SplitAssignment> {
+    async fn reassign_splits(&self) -> MetaResult<HashMap<DatabaseId, SplitAssignment>> {
         let mut split_assignment: SplitAssignment = HashMap::new();
 
         for (source_id, handle) in &self.managed_sources {
@@ -421,7 +420,9 @@ impl SourceManagerCore {
             }
         }
 
-        Ok(split_assignment)
+        self.metadata_manager
+            .split_fragment_map_by_database(split_assignment)
+            .await
     }
 
     fn apply_source_change(
@@ -1001,6 +1002,7 @@ impl SourceManager {
 
     /// create and register connector worker for source.
     pub async fn register_source(&self, source: &Source) -> MetaResult<()> {
+        tracing::debug!("register_source: {}", source.get_id());
         let mut core = self.core.lock().await;
         if let Entry::Vacant(e) = core.managed_sources.entry(source.get_id() as _) {
             let handle = create_source_worker_handle(source, self.metrics.clone())
@@ -1050,6 +1052,7 @@ impl SourceManager {
         let source_id = source.id;
 
         let connector_properties = extract_prop_from_existing_source(&source)?;
+
         let enable_scale_in = connector_properties.enable_split_scale_in();
         let (sync_call_tx, sync_call_rx) = tokio::sync::mpsc::unbounded_channel();
         let handle = tokio::spawn(async move {
@@ -1122,10 +1125,17 @@ impl SourceManager {
             core_guard.reassign_splits().await?
         };
 
-        if !split_assignment.is_empty() {
-            let command = Command::SourceSplitAssignment(split_assignment);
-            tracing::info!(command = ?command, "pushing down split assignment command");
-            self.barrier_scheduler.run_command(command).await?;
+        for (database_id, split_assignment) in split_assignment {
+            if !split_assignment.is_empty() {
+                let command = Command::SourceSplitAssignment(split_assignment);
+                tracing::info!(command = ?command, "pushing down split assignment command");
+                self.barrier_scheduler
+                    .run_command(
+                        risingwave_common::catalog::DatabaseId::new(database_id as _),
+                        command,
+                    )
+                    .await?;
+            }
         }
 
         Ok(())
diff --git a/src/meta/src/stream/stream_graph/actor.rs b/src/meta/src/stream/stream_graph/actor.rs
index 3446a2661d962..d7e1b0b1b6df7 100644
--- a/src/meta/src/stream/stream_graph/actor.rs
+++ b/src/meta/src/stream/stream_graph/actor.rs
@@ -20,7 +20,7 @@ use assert_matches::assert_matches;
 use itertools::Itertools;
 use risingwave_common::bail;
 use risingwave_common::bitmap::Bitmap;
-use risingwave_common::hash::{ActorId, ActorMapping, WorkerSlotId};
+use risingwave_common::hash::{ActorId, ActorMapping, IsSingleton, VnodeCount, WorkerSlotId};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::stream_graph_visitor::visit_tables;
 use risingwave_meta_model::WorkerId;
@@ -683,9 +683,22 @@ impl ActorGraphBuilder {
         // Fill the vnode count for each internal table, based on schedule result.
         let mut fragment_graph = fragment_graph;
         for (id, fragment) in fragment_graph.building_fragments_mut() {
-            let vnode_count = distributions[id].vnode_count();
+            let fragment_vnode_count = distributions[id].vnode_count();
             visit_tables(fragment, |table, _| {
-                table.maybe_vnode_count = Some(vnode_count as _);
+                // There are special cases where a hash-distributed fragment contains singleton
+                // internal tables, e.g., the state table of `Source` executors.
+                let vnode_count = if table.is_singleton() {
+                    if fragment_vnode_count > 1 {
+                        tracing::info!(
+                            table.name,
+                            "found singleton table in hash-distributed fragment"
+                        );
+                    }
+                    1
+                } else {
+                    fragment_vnode_count
+                };
+                table.maybe_vnode_count = VnodeCount::set(vnode_count).to_protobuf();
             })
         }
 
diff --git a/src/meta/src/stream/stream_graph/fragment.rs b/src/meta/src/stream/stream_graph/fragment.rs
index aaa30ee0d57c5..c2ccd4300ccf1 100644
--- a/src/meta/src/stream/stream_graph/fragment.rs
+++ b/src/meta/src/stream/stream_graph/fragment.rs
@@ -24,6 +24,7 @@ use risingwave_common::bail;
 use risingwave_common::catalog::{
     generate_internal_table_name_with_type, TableId, CDC_SOURCE_COLUMN_NUM,
 };
+use risingwave_common::hash::VnodeCount;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common::util::stream_graph_visitor;
 use risingwave_common::util::stream_graph_visitor::visit_stream_node_cont;
@@ -55,7 +56,7 @@ pub(super) struct BuildingFragment {
     inner: StreamFragment,
 
     /// The ID of the job if it contains the streaming job node.
-    table_id: Option<u32>,
+    job_id: Option<u32>,
 
     /// The required column IDs of each upstream table.
     /// Will be converted to indices when building the edge connected to the upstream.
@@ -81,12 +82,12 @@ impl BuildingFragment {
         // Fill the information of the internal tables in the fragment.
         Self::fill_internal_tables(&mut fragment, job, table_id_gen);
 
-        let table_id = Self::fill_job(&mut fragment, job).then(|| job.id());
+        let job_id = Self::fill_job(&mut fragment, job).then(|| job.id());
         let upstream_table_columns = Self::extract_upstream_table_columns(&mut fragment);
 
         Self {
             inner: fragment,
-            table_id,
+            job_id,
             upstream_table_columns,
         }
     }
@@ -125,17 +126,17 @@ impl BuildingFragment {
 
     /// Fill the information with the job in the fragment.
     fn fill_job(fragment: &mut StreamFragment, job: &StreamingJob) -> bool {
-        let table_id = job.id();
+        let job_id = job.id();
         let fragment_id = fragment.fragment_id;
-        let mut has_table = false;
+        let mut has_job = false;
 
         stream_graph_visitor::visit_fragment(fragment, |node_body| match node_body {
             NodeBody::Materialize(materialize_node) => {
-                materialize_node.table_id = table_id;
+                materialize_node.table_id = job_id;
 
                 // Fill the ID of the `Table`.
                 let table = materialize_node.table.as_mut().unwrap();
-                table.id = table_id;
+                table.id = job_id;
                 table.database_id = job.database_id();
                 table.schema_id = job.schema_id();
                 table.fragment_id = fragment_id;
@@ -144,27 +145,49 @@ impl BuildingFragment {
                     table.definition = job.name();
                 }
 
-                has_table = true;
+                has_job = true;
             }
             NodeBody::Sink(sink_node) => {
-                sink_node.sink_desc.as_mut().unwrap().id = table_id;
+                sink_node.sink_desc.as_mut().unwrap().id = job_id;
 
-                has_table = true;
+                has_job = true;
             }
             NodeBody::Dml(dml_node) => {
-                dml_node.table_id = table_id;
+                dml_node.table_id = job_id;
                 dml_node.table_version_id = job.table_version_id().unwrap();
             }
-            NodeBody::Source(_) => {
-                // Notice: Table job has a dumb Source node, we should be careful that `has_table` should not be overwrite to `false`
-                if !has_table {
-                    has_table = job.is_source_job();
+            NodeBody::Source(source_node) => {
+                match job {
+                    // Note: For table without connector, it has a dummy Source node.
+                    // Note: For table with connector, it's source node has a source id different with the table id (job id), assigned in create_job_catalog.
+                    StreamingJob::Table(source, _table, _table_job_type) => {
+                        if let Some(source_inner) = source_node.source_inner.as_mut() {
+                            if let Some(source) = source {
+                                debug_assert_ne!(source.id, job_id);
+                                source_inner.source_id = source.id;
+                            }
+                        }
+                    }
+                    StreamingJob::Source(source) => {
+                        has_job = true;
+                        if let Some(source_inner) = source_node.source_inner.as_mut() {
+                            debug_assert_eq!(source.id, job_id);
+                            source_inner.source_id = source.id;
+                        }
+                    }
+                    // For other job types, no need to fill the source id, since it refers to an existing source.
+                    _ => {}
+                }
+            }
+            NodeBody::StreamCdcScan(node) => {
+                if let Some(table_desc) = node.cdc_table_desc.as_mut() {
+                    table_desc.table_id = job_id;
                 }
             }
             _ => {}
         });
 
-        has_table
+        has_job
     }
 
     /// Extract the required columns (in IDs) of each upstream table.
@@ -349,6 +372,9 @@ impl StreamFragmentGraph {
     ) -> MetaResult<Self> {
         let fragment_id_gen =
             GlobalFragmentIdGen::new(env.id_gen_manager(), proto.fragments.len() as u64);
+        // Note: in SQL backend, the ids generated here are fake and will be overwritten again
+        // with `refill_internal_table_ids` later.
+        // TODO: refactor the code to remove this step.
         let table_id_gen = GlobalTableIdGen::new(env.id_gen_manager(), proto.table_ids_cnt as u64);
 
         // Create nodes.
@@ -417,8 +443,14 @@ impl StreamFragmentGraph {
         })
     }
 
-    /// Retrieve the internal tables map of the whole graph.
-    pub fn internal_tables(&self) -> BTreeMap<u32, Table> {
+    /// Retrieve the **incomplete** internal tables map of the whole graph.
+    ///
+    /// Note that some fields in the table catalogs are not filled during the current phase, e.g.,
+    /// `fragment_id`, `vnode_count`. They will be all filled after a `TableFragments` is built.
+    /// Be careful when using the returned values.
+    ///
+    /// See also [`crate::model::TableFragments::internal_tables`].
+    pub fn incomplete_internal_tables(&self) -> BTreeMap<u32, Table> {
         let mut tables = BTreeMap::new();
         for fragment in self.fragments.values() {
             for table in fragment.extract_internal_tables() {
@@ -431,6 +463,8 @@ impl StreamFragmentGraph {
         tables
     }
 
+    /// Refill the internal tables' `table_id`s according to the given map, typically obtained from
+    /// `create_internal_table_catalog`.
     pub fn refill_internal_table_ids(&mut self, table_id_map: HashMap<u32, u32>) {
         for fragment in self.fragments.values_mut() {
             stream_graph_visitor::visit_internal_tables(
@@ -487,7 +521,7 @@ impl StreamFragmentGraph {
     pub fn table_fragment_id(&self) -> FragmentId {
         self.fragments
             .values()
-            .filter(|b| b.table_id.is_some())
+            .filter(|b| b.job_id.is_some())
             .map(|b| b.fragment_id)
             .exactly_one()
             .expect("require exactly 1 materialize/sink/cdc source node when creating the streaming job")
@@ -1083,7 +1117,7 @@ impl CompleteStreamFragmentGraph {
         let internal_tables = building_fragment.extract_internal_tables();
         let BuildingFragment {
             inner,
-            table_id,
+            job_id,
             upstream_table_columns: _,
         } = building_fragment;
 
@@ -1092,7 +1126,7 @@ impl CompleteStreamFragmentGraph {
 
         let materialized_fragment_id =
             if inner.fragment_type_mask & FragmentTypeFlag::Mview as u32 != 0 {
-                table_id
+                job_id
             } else {
                 None
             };
@@ -1115,7 +1149,7 @@ impl CompleteStreamFragmentGraph {
             actors,
             state_table_ids,
             upstream_fragment_ids,
-            maybe_vnode_count: Some(vnode_count as _),
+            maybe_vnode_count: VnodeCount::set(vnode_count).to_protobuf(),
         }
     }
 
diff --git a/src/meta/src/stream/stream_graph/schedule.rs b/src/meta/src/stream/stream_graph/schedule.rs
index 5d465b19d195d..97b5c3032117e 100644
--- a/src/meta/src/stream/stream_graph/schedule.rs
+++ b/src/meta/src/stream/stream_graph/schedule.rs
@@ -25,7 +25,7 @@ use either::Either;
 use enum_as_inner::EnumAsInner;
 use itertools::Itertools;
 use risingwave_common::bitmap::Bitmap;
-use risingwave_common::hash::{ActorMapping, VirtualNode, WorkerSlotId, WorkerSlotMapping};
+use risingwave_common::hash::{ActorMapping, WorkerSlotId, WorkerSlotMapping};
 use risingwave_common::{bail, hash};
 use risingwave_meta_model::WorkerId;
 use risingwave_pb::common::{ActorInfo, WorkerNode};
@@ -152,11 +152,9 @@ impl Distribution {
     }
 
     /// Get the vnode count of the distribution.
-    ///
-    /// For backwards compatibility, [`VirtualNode::COUNT_FOR_COMPAT`] is used for singleton.
     pub fn vnode_count(&self) -> usize {
         match self {
-            Distribution::Singleton(_) => VirtualNode::COUNT_FOR_COMPAT,
+            Distribution::Singleton(_) => 1, // only `SINGLETON_VNODE`
             Distribution::Hash(mapping) => mapping.len(),
         }
     }
diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs
index 21a642f1be1c7..509fffefd99b9 100644
--- a/src/meta/src/stream/stream_manager.rs
+++ b/src/meta/src/stream/stream_manager.rs
@@ -18,7 +18,7 @@ use std::sync::Arc;
 use futures::future::join_all;
 use itertools::Itertools;
 use risingwave_common::bail;
-use risingwave_common::catalog::TableId;
+use risingwave_common::catalog::{DatabaseId, TableId};
 use risingwave_meta_model::{ObjectId, WorkerId};
 use risingwave_pb::catalog::{CreateType, Subscription, Table};
 use risingwave_pb::stream_plan::update_mutation::MergeUpdate;
@@ -238,6 +238,7 @@ impl GlobalStreamManager {
         ctx: CreateStreamingJobContext,
     ) -> MetaResult<NotificationVersion> {
         let table_id = table_fragments.table_id();
+        let database_id = ctx.streaming_job.database_id().into();
         let (sender, mut receiver) = tokio::sync::mpsc::channel(10);
         let execution = StreamingJobExecution::new(table_id, sender.clone());
         self.creating_job_info.add_job(execution).await;
@@ -298,7 +299,10 @@ impl GlobalStreamManager {
                                 .await?;
 
                             self.barrier_scheduler
-                                .run_command(Command::CancelStreamingJob(table_fragments))
+                                .run_command(
+                                    database_id,
+                                    Command::CancelStreamingJob(table_fragments),
+                                )
                                 .await?;
                         } else {
                             // streaming job is already completed.
@@ -350,7 +354,7 @@ impl GlobalStreamManager {
         if let Some((streaming_job, context, table_fragments)) = replace_table_job_info {
             self.metadata_manager
                 .catalog_controller
-                .prepare_streaming_job(table_fragments.to_protobuf(), &streaming_job, true)
+                .prepare_streaming_job(&table_fragments, &streaming_job, true)
                 .await?;
 
             let dummy_table_id = table_fragments.table_id();
@@ -422,10 +426,15 @@ impl GlobalStreamManager {
             if need_pause {
                 // Special handling is required when creating sink into table, we need to pause the stream to avoid data loss.
                 self.barrier_scheduler
-                    .run_config_change_command_with_pause(command)
+                    .run_config_change_command_with_pause(
+                        streaming_job.database_id().into(),
+                        command,
+                    )
                     .await?;
             } else {
-                self.barrier_scheduler.run_command(command).await?;
+                self.barrier_scheduler
+                    .run_command(streaming_job.database_id().into(), command)
+                    .await?;
             }
 
             tracing::debug!(?streaming_job, "first barrier collected for stream job");
@@ -468,15 +477,18 @@ impl GlobalStreamManager {
         let init_split_assignment = self.source_manager.allocate_splits(&dummy_table_id).await?;
 
         self.barrier_scheduler
-            .run_config_change_command_with_pause(Command::ReplaceTable(ReplaceTablePlan {
-                old_table_fragments,
-                new_table_fragments: table_fragments,
-                merge_updates,
-                dispatchers,
-                init_split_assignment,
-                dummy_id,
-                streaming_job,
-            }))
+            .run_config_change_command_with_pause(
+                streaming_job.database_id().into(),
+                Command::ReplaceTable(ReplaceTablePlan {
+                    old_table_fragments,
+                    new_table_fragments: table_fragments,
+                    merge_updates,
+                    dispatchers,
+                    init_split_assignment,
+                    dummy_id,
+                    streaming_job,
+                }),
+            )
             .await?;
 
         Ok(())
@@ -487,6 +499,7 @@ impl GlobalStreamManager {
     /// [`Command::DropStreamingJobs`] for details.
     pub async fn drop_streaming_jobs(
         &self,
+        database_id: DatabaseId,
         removed_actors: Vec<ActorId>,
         streaming_job_ids: Vec<ObjectId>,
         state_table_ids: Vec<risingwave_meta_model::TableId>,
@@ -498,14 +511,17 @@ impl GlobalStreamManager {
         {
             let _ = self
                 .barrier_scheduler
-                .run_command(Command::DropStreamingJobs {
-                    actors: removed_actors,
-                    unregistered_state_table_ids: state_table_ids
-                        .into_iter()
-                        .map(|table_id| TableId::new(table_id as _))
-                        .collect(),
-                    unregistered_fragment_ids: fragment_ids,
-                })
+                .run_command(
+                    database_id,
+                    Command::DropStreamingJobs {
+                        actors: removed_actors,
+                        unregistered_state_table_ids: state_table_ids
+                            .into_iter()
+                            .map(|table_id| TableId::new(table_id as _))
+                            .collect(),
+                        unregistered_fragment_ids: fragment_ids,
+                    },
+                )
                 .await
                 .inspect_err(|err| {
                     tracing::error!(error = ?err.as_report(), "failed to run drop command");
@@ -553,14 +569,16 @@ impl GlobalStreamManager {
                     )))?;
                 }
 
-                self.metadata_manager
+                let (_, database_id) = self.metadata_manager
                     .catalog_controller
                     .try_abort_creating_streaming_job(id.table_id as _, true)
                     .await?;
 
-                self.barrier_scheduler
-                    .run_command(Command::CancelStreamingJob(fragment))
-                    .await?;
+                if let Some(database_id) = database_id {
+                    self.barrier_scheduler
+                        .run_command(DatabaseId::new(database_id as _), Command::CancelStreamingJob(fragment))
+                        .await?;
+                }
             };
             match result {
                 Ok(_) => {
@@ -587,6 +605,12 @@ impl GlobalStreamManager {
     ) -> MetaResult<()> {
         let _reschedule_job_lock = self.reschedule_lock_write_guard().await;
 
+        let database_id = DatabaseId::new(
+            self.metadata_manager
+                .catalog_controller
+                .get_object_database_id(table_id as ObjectId)
+                .await? as _,
+        );
         let table_id = TableId::new(table_id);
 
         let worker_nodes = self
@@ -665,6 +689,7 @@ impl GlobalStreamManager {
                     .await?;
             } else {
                 self.reschedule_actors(
+                    database_id,
                     reschedules,
                     RescheduleOptions {
                         resolve_no_shuffle_upstream: false,
@@ -691,12 +716,19 @@ impl GlobalStreamManager {
         };
 
         tracing::debug!("sending Command::CreateSubscription");
-        self.barrier_scheduler.run_command(command).await?;
+        self.barrier_scheduler
+            .run_command(subscription.database_id.into(), command)
+            .await?;
         Ok(())
     }
 
     // Don't need to add actor, just send a command
-    pub async fn drop_subscription(self: &Arc<Self>, subscription_id: u32, table_id: u32) {
+    pub async fn drop_subscription(
+        self: &Arc<Self>,
+        database_id: DatabaseId,
+        subscription_id: u32,
+        table_id: u32,
+    ) {
         let command = Command::DropSubscription {
             subscription_id,
             upstream_mv_table_id: TableId::new(table_id),
@@ -705,7 +737,7 @@ impl GlobalStreamManager {
         tracing::debug!("sending Command::DropSubscriptions");
         let _ = self
             .barrier_scheduler
-            .run_command(command)
+            .run_command(database_id, command)
             .await
             .inspect_err(|err| {
                 tracing::error!(error = ?err.as_report(), "failed to run drop command");
diff --git a/src/meta/src/stream/test_fragmenter.rs b/src/meta/src/stream/test_fragmenter.rs
index 9818af124537e..7c7aed920ca3b 100644
--- a/src/meta/src/stream/test_fragmenter.rs
+++ b/src/meta/src/stream/test_fragmenter.rs
@@ -448,7 +448,7 @@ async fn test_graph_builder() -> MetaResult<()> {
         time_zone: graph.ctx.as_ref().unwrap().timezone.clone(),
     };
     let fragment_graph = StreamFragmentGraph::new(&env, graph, &job)?;
-    let internal_tables = fragment_graph.internal_tables();
+    let internal_tables = fragment_graph.incomplete_internal_tables();
 
     let actor_graph_builder = ActorGraphBuilder::new(
         job.id(),
@@ -461,17 +461,9 @@ async fn test_graph_builder() -> MetaResult<()> {
 
     let table_fragments = TableFragments::for_test(TableId::default(), graph);
     let actors = table_fragments.actors();
-    let barrier_inject_actor_ids = table_fragments
-        .fragments
-        .values()
-        .filter(|fragment| TableFragments::is_injectable(fragment.fragment_type_mask))
-        .flat_map(|fragment| fragment.actors.iter().map(|actor| actor.actor_id))
-        .sorted()
-        .collect_vec();
     let mview_actor_ids = table_fragments.mview_actor_ids();
 
     assert_eq!(actors.len(), 9);
-    assert_eq!(barrier_inject_actor_ids, vec![6, 7, 8, 9]);
     assert_eq!(mview_actor_ids, vec![1]);
     assert_eq!(internal_tables.len(), 3);
 
diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml
index 912eb4073130e..54bc6b4524f66 100644
--- a/src/object_store/Cargo.toml
+++ b/src/object_store/Cargo.toml
@@ -30,7 +30,7 @@ hyper = { version = "0.14", features = ["tcp", "client"] }                    #
 hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] }
 hyper-tls = "0.5.0"
 itertools = { workspace = true }
-madsim = "0.2.30"
+madsim = "0.2.31"
 opendal = { workspace = true, features = [
     "executors-tokio",
     "services-azblob",
@@ -47,7 +47,6 @@ prometheus = { version = "0.13", features = ["process"] }
 reqwest = "0.12.2" # required by opendal
 risingwave_common = { workspace = true }
 risingwave_jni_core = { workspace = true, optional = true }
-rustls = "0.23.5"
 spin = "0.9"
 thiserror = "1"
 thiserror-ext = { workspace = true }
diff --git a/src/prost/build.rs b/src/prost/build.rs
index 194d565e31508..c4744e14c1b60 100644
--- a/src/prost/build.rs
+++ b/src/prost/build.rs
@@ -183,6 +183,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
         .type_attribute("hummock.GroupTableChange", "#[derive(Eq)]")
         .type_attribute("hummock.GroupMerge", "#[derive(Eq)]")
         .type_attribute("hummock.GroupDelta", "#[derive(Eq)]")
+        .type_attribute("hummock.NewL0SubLevel", "#[derive(Eq)]")
         .type_attribute("hummock.LevelHandler.RunningCompactTask", "#[derive(Eq)]")
         .type_attribute("hummock.LevelHandler", "#[derive(Eq)]")
         .type_attribute("hummock.TableOption", "#[derive(Eq)]")
diff --git a/src/risedevtool/src/risedev_env.rs b/src/risedevtool/src/risedev_env.rs
index 4860d30e01485..5d7c113b6bc30 100644
--- a/src/risedevtool/src/risedev_env.rs
+++ b/src/risedevtool/src/risedev_env.rs
@@ -105,7 +105,7 @@ pub fn generate_risedev_env(services: &Vec<ServiceConfig>) -> String {
                 writeln!(env, r#"PUBSUB_EMULATOR_HOST="{address}:{port}""#,).unwrap();
                 writeln!(env, r#"RISEDEV_PUBSUB_WITH_OPTIONS_COMMON="connector='google_pubsub',pubsub.emulator_host='{address}:{port}'""#,).unwrap();
             }
-            ServiceConfig::Postgres(c) if c.application != Application::Metastore => {
+            ServiceConfig::Postgres(c) => {
                 let host = &c.address;
                 let port = &c.port;
                 let user = &c.user;
diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml
index 6a25be3c21738..f7010f975e692 100644
--- a/src/rpc_client/Cargo.toml
+++ b/src/rpc_client/Cargo.toml
@@ -45,7 +45,7 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [
 tokio-retry = "0.3"
 tokio-stream = { workspace = true }
 tonic = { workspace = true }
-tower = "0.4"
+tower = "0.5"
 tracing = "0.1"
 url = "2.5.0"
 
diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs
index e8b46ded0ed19..24375d395a069 100644
--- a/src/rpc_client/src/meta_client.rs
+++ b/src/rpc_client/src/meta_client.rs
@@ -573,6 +573,19 @@ impl MetaClient {
         Ok(())
     }
 
+    pub async fn alter_swap_rename(
+        &self,
+        object: alter_swap_rename_request::Object,
+    ) -> Result<WaitVersion> {
+        let request = AlterSwapRenameRequest {
+            object: Some(object),
+        };
+        let resp = self.inner.alter_swap_rename(request).await?;
+        Ok(resp
+            .version
+            .ok_or_else(|| anyhow!("wait version not set"))?)
+    }
+
     pub async fn replace_table(
         &self,
         source: Option<PbSource>,
@@ -905,8 +918,8 @@ impl MetaClient {
         Ok(resp.tables)
     }
 
-    pub async fn flush(&self, checkpoint: bool) -> Result<HummockVersionId> {
-        let request = FlushRequest { checkpoint };
+    pub async fn flush(&self, database_id: DatabaseId) -> Result<HummockVersionId> {
+        let request = FlushRequest { database_id };
         let resp = self.inner.flush(request).await?;
         Ok(HummockVersionId::new(resp.hummock_version_id))
     }
@@ -1423,6 +1436,26 @@ impl MetaClient {
         .join();
     }
 
+    pub async fn add_sink_fail_evet(
+        &self,
+        sink_id: u32,
+        sink_name: String,
+        connector: String,
+        error: String,
+    ) -> Result<()> {
+        let event = event_log::EventSinkFail {
+            sink_id,
+            sink_name,
+            connector,
+            error,
+        };
+        let req = AddEventLogRequest {
+            event: Some(add_event_log_request::Event::SinkFail(event)),
+        };
+        self.inner.add_event_log(req).await?;
+        Ok(())
+    }
+
     pub async fn cancel_compact_task(&self, task_id: u64, task_status: TaskStatus) -> Result<bool> {
         let req = CancelCompactTaskRequest {
             task_id,
@@ -2076,6 +2109,7 @@ macro_rules! for_all_meta_rpc {
             ,{ ddl_client, get_tables, GetTablesRequest, GetTablesResponse }
             ,{ ddl_client, wait, WaitRequest, WaitResponse }
             ,{ ddl_client, auto_schema_change, AutoSchemaChangeRequest, AutoSchemaChangeResponse }
+            ,{ ddl_client, alter_swap_rename, AlterSwapRenameRequest, AlterSwapRenameResponse }
             ,{ hummock_client, unpin_version_before, UnpinVersionBeforeRequest, UnpinVersionBeforeResponse }
             ,{ hummock_client, get_current_version, GetCurrentVersionRequest, GetCurrentVersionResponse }
             ,{ hummock_client, replay_version_delta, ReplayVersionDeltaRequest, ReplayVersionDeltaResponse }
diff --git a/src/sqlparser/src/ast/ddl.rs b/src/sqlparser/src/ast/ddl.rs
index 89e8f24bf5922..d94cf80cb9f2d 100644
--- a/src/sqlparser/src/ast/ddl.rs
+++ b/src/sqlparser/src/ast/ddl.rs
@@ -20,7 +20,7 @@ use core::fmt;
 #[cfg(feature = "serde")]
 use serde::{Deserialize, Serialize};
 
-use super::ConnectorSchema;
+use super::FormatEncodeOptions;
 use crate::ast::{
     display_comma_separated, display_separated, DataType, Expr, Ident, ObjectName, SetVariableValue,
 };
@@ -38,6 +38,7 @@ pub enum AlterDatabaseOperation {
 pub enum AlterSchemaOperation {
     ChangeOwner { new_owner_name: Ident },
     RenameSchema { schema_name: ObjectName },
+    SwapRenameSchema { target_schema: ObjectName },
 }
 
 /// An `ALTER TABLE` (`Statement::AlterTable`) operation
@@ -110,6 +111,10 @@ pub enum AlterTableOperation {
     SetBackfillRateLimit {
         rate_limit: i32,
     },
+    /// `SWAP WITH <table_name>`
+    SwapRenameTable {
+        target_table: ObjectName,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -146,6 +151,10 @@ pub enum AlterViewOperation {
     SetBackfillRateLimit {
         rate_limit: i32,
     },
+    /// `SWAP WITH <view_name>`
+    SwapRenameView {
+        target_view: ObjectName,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -165,6 +174,10 @@ pub enum AlterSinkOperation {
         parallelism: SetVariableValue,
         deferred: bool,
     },
+    /// `SWAP WITH <sink_name>`
+    SwapRenameSink {
+        target_sink: ObjectName,
+    },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -173,6 +186,7 @@ pub enum AlterSubscriptionOperation {
     RenameSubscription { subscription_name: ObjectName },
     ChangeOwner { new_owner_name: Ident },
     SetSchema { new_schema_name: ObjectName },
+    SwapRenameSubscription { target_subscription: ObjectName },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -182,9 +196,10 @@ pub enum AlterSourceOperation {
     AddColumn { column_def: ColumnDef },
     ChangeOwner { new_owner_name: Ident },
     SetSchema { new_schema_name: ObjectName },
-    FormatEncode { connector_schema: ConnectorSchema },
+    FormatEncode { format_encode: FormatEncodeOptions },
     RefreshSchema,
     SetSourceRateLimit { rate_limit: i32 },
+    SwapRenameSource { target_source: ObjectName },
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -221,6 +236,9 @@ impl fmt::Display for AlterSchemaOperation {
             AlterSchemaOperation::RenameSchema { schema_name } => {
                 write!(f, "RENAME TO {}", schema_name)
             }
+            AlterSchemaOperation::SwapRenameSchema { target_schema } => {
+                write!(f, "SWAP WITH {}", target_schema)
+            }
         }
     }
 }
@@ -300,6 +318,9 @@ impl fmt::Display for AlterTableOperation {
             AlterTableOperation::SetBackfillRateLimit { rate_limit } => {
                 write!(f, "SET BACKFILL_RATE_LIMIT TO {}", rate_limit)
             }
+            AlterTableOperation::SwapRenameTable { target_table } => {
+                write!(f, "SWAP WITH {}", target_table)
+            }
         }
     }
 }
@@ -351,6 +372,9 @@ impl fmt::Display for AlterViewOperation {
             AlterViewOperation::SetBackfillRateLimit { rate_limit } => {
                 write!(f, "SET BACKFILL_RATE_LIMIT TO {}", rate_limit)
             }
+            AlterViewOperation::SwapRenameView { target_view } => {
+                write!(f, "SWAP WITH {}", target_view)
+            }
         }
     }
 }
@@ -378,6 +402,9 @@ impl fmt::Display for AlterSinkOperation {
                     if *deferred { " DEFERRED" } else { "" }
                 )
             }
+            AlterSinkOperation::SwapRenameSink { target_sink } => {
+                write!(f, "SWAP WITH {}", target_sink)
+            }
         }
     }
 }
@@ -394,6 +421,11 @@ impl fmt::Display for AlterSubscriptionOperation {
             AlterSubscriptionOperation::SetSchema { new_schema_name } => {
                 write!(f, "SET SCHEMA {}", new_schema_name)
             }
+            AlterSubscriptionOperation::SwapRenameSubscription {
+                target_subscription,
+            } => {
+                write!(f, "SWAP WITH {}", target_subscription)
+            }
         }
     }
 }
@@ -413,8 +445,8 @@ impl fmt::Display for AlterSourceOperation {
             AlterSourceOperation::SetSchema { new_schema_name } => {
                 write!(f, "SET SCHEMA {}", new_schema_name)
             }
-            AlterSourceOperation::FormatEncode { connector_schema } => {
-                write!(f, "{connector_schema}")
+            AlterSourceOperation::FormatEncode { format_encode } => {
+                write!(f, "{format_encode}")
             }
             AlterSourceOperation::RefreshSchema => {
                 write!(f, "REFRESH SCHEMA")
@@ -422,6 +454,9 @@ impl fmt::Display for AlterSourceOperation {
             AlterSourceOperation::SetSourceRateLimit { rate_limit } => {
                 write!(f, "SET SOURCE_RATE_LIMIT TO {}", rate_limit)
             }
+            AlterSourceOperation::SwapRenameSource { target_source } => {
+                write!(f, "SWAP WITH {}", target_source)
+            }
         }
     }
 }
@@ -731,7 +766,7 @@ impl fmt::Display for ColumnOption {
 
 fn display_constraint_name(name: &'_ Option<Ident>) -> impl fmt::Display + '_ {
     struct ConstraintName<'a>(&'a Option<Ident>);
-    impl<'a> fmt::Display for ConstraintName<'a> {
+    impl fmt::Display for ConstraintName<'_> {
         fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
             if let Some(name) = self.0 {
                 write!(f, "CONSTRAINT {} ", name)?;
diff --git a/src/sqlparser/src/ast/legacy_source.rs b/src/sqlparser/src/ast/legacy_source.rs
index 5fb2e233a67df..7a5abf35a2df8 100644
--- a/src/sqlparser/src/ast/legacy_source.rs
+++ b/src/sqlparser/src/ast/legacy_source.rs
@@ -23,7 +23,7 @@ use serde::{Deserialize, Serialize};
 use winnow::PResult;
 
 use crate::ast::{
-    display_separated, AstString, ConnectorSchema, Encode, Format, Ident, ObjectName, ParseTo,
+    display_separated, AstString, Encode, Format, FormatEncodeOptions, Ident, ObjectName, ParseTo,
     SqlOption, Value,
 };
 use crate::keywords::Keyword;
@@ -32,45 +32,45 @@ use crate::{impl_fmt_display, impl_parse_to, parser_err};
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub enum CompatibleSourceSchema {
-    RowFormat(SourceSchema),
-    V2(ConnectorSchema),
+pub enum CompatibleFormatEncode {
+    RowFormat(LegacyRowFormat),
+    V2(FormatEncodeOptions),
 }
 
-impl fmt::Display for CompatibleSourceSchema {
+impl fmt::Display for CompatibleFormatEncode {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            CompatibleSourceSchema::RowFormat(inner) => {
+            CompatibleFormatEncode::RowFormat(inner) => {
                 write!(f, "{}", inner)
             }
-            CompatibleSourceSchema::V2(inner) => {
+            CompatibleFormatEncode::V2(inner) => {
                 write!(f, "{}", inner)
             }
         }
     }
 }
 
-impl CompatibleSourceSchema {
-    pub(crate) fn into_v2(self) -> ConnectorSchema {
+impl CompatibleFormatEncode {
+    pub(crate) fn into_v2(self) -> FormatEncodeOptions {
         match self {
-            CompatibleSourceSchema::RowFormat(inner) => inner.into_source_schema_v2(),
-            CompatibleSourceSchema::V2(inner) => inner,
+            CompatibleFormatEncode::RowFormat(inner) => inner.into_format_encode_v2(),
+            CompatibleFormatEncode::V2(inner) => inner,
         }
     }
 }
 
-impl From<ConnectorSchema> for CompatibleSourceSchema {
-    fn from(value: ConnectorSchema) -> Self {
+impl From<FormatEncodeOptions> for CompatibleFormatEncode {
+    fn from(value: FormatEncodeOptions) -> Self {
         Self::V2(value)
     }
 }
 
-pub fn parse_source_schema(p: &mut Parser<'_>) -> PResult<CompatibleSourceSchema> {
+pub fn parse_format_encode(p: &mut Parser<'_>) -> PResult<CompatibleFormatEncode> {
     if let Some(schema_v2) = p.parse_schema()? {
         if schema_v2.key_encode.is_some() {
             parser_err!("key encode clause is not supported in source schema");
         }
-        Ok(CompatibleSourceSchema::V2(schema_v2))
+        Ok(CompatibleFormatEncode::V2(schema_v2))
     } else if p.peek_nth_any_of_keywords(0, &[Keyword::ROW])
         && p.peek_nth_any_of_keywords(1, &[Keyword::FORMAT])
     {
@@ -79,34 +79,34 @@ pub fn parse_source_schema(p: &mut Parser<'_>) -> PResult<CompatibleSourceSchema
         let id = p.parse_identifier()?;
         let value = id.value.to_ascii_uppercase();
         let schema = match &value[..] {
-            "JSON" => SourceSchema::Json,
-            "UPSERT_JSON" => SourceSchema::UpsertJson,
+            "JSON" => LegacyRowFormat::Json,
+            "UPSERT_JSON" => LegacyRowFormat::UpsertJson,
             "PROTOBUF" => {
                 impl_parse_to!(protobuf_schema: ProtobufSchema, p);
-                SourceSchema::Protobuf(protobuf_schema)
+                LegacyRowFormat::Protobuf(protobuf_schema)
             }
-            "DEBEZIUM_JSON" => SourceSchema::DebeziumJson,
-            "DEBEZIUM_MONGO_JSON" => SourceSchema::DebeziumMongoJson,
+            "DEBEZIUM_JSON" => LegacyRowFormat::DebeziumJson,
+            "DEBEZIUM_MONGO_JSON" => LegacyRowFormat::DebeziumMongoJson,
             "AVRO" => {
                 impl_parse_to!(avro_schema: AvroSchema, p);
-                SourceSchema::Avro(avro_schema)
+                LegacyRowFormat::Avro(avro_schema)
             }
             "UPSERT_AVRO" => {
                 impl_parse_to!(avro_schema: AvroSchema, p);
-                SourceSchema::UpsertAvro(avro_schema)
+                LegacyRowFormat::UpsertAvro(avro_schema)
             }
-            "MAXWELL" => SourceSchema::Maxwell,
-            "CANAL_JSON" => SourceSchema::CanalJson,
+            "MAXWELL" => LegacyRowFormat::Maxwell,
+            "CANAL_JSON" => LegacyRowFormat::CanalJson,
             "CSV" => {
                 impl_parse_to!(csv_info: CsvInfo, p);
-                SourceSchema::Csv(csv_info)
+                LegacyRowFormat::Csv(csv_info)
             }
-            "NATIVE" => SourceSchema::Native, // used internally by schema change
+            "NATIVE" => LegacyRowFormat::Native, // used internally by schema change
             "DEBEZIUM_AVRO" => {
                 impl_parse_to!(avro_schema: DebeziumAvroSchema, p);
-                SourceSchema::DebeziumAvro(avro_schema)
+                LegacyRowFormat::DebeziumAvro(avro_schema)
             }
-            "BYTES" => SourceSchema::Bytes,
+            "BYTES" => LegacyRowFormat::Bytes,
             _ => {
                 parser_err!(
                     "expected JSON | UPSERT_JSON | PROTOBUF | DEBEZIUM_JSON | DEBEZIUM_AVRO \
@@ -114,7 +114,7 @@ pub fn parse_source_schema(p: &mut Parser<'_>) -> PResult<CompatibleSourceSchema
                 );
             }
         };
-        Ok(CompatibleSourceSchema::RowFormat(schema))
+        Ok(CompatibleFormatEncode::RowFormat(schema))
     } else {
         p.expected("description of the format")
     }
@@ -122,7 +122,7 @@ pub fn parse_source_schema(p: &mut Parser<'_>) -> PResult<CompatibleSourceSchema
 
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub enum SourceSchema {
+pub enum LegacyRowFormat {
     Protobuf(ProtobufSchema), // Keyword::PROTOBUF ProtobufSchema
     Json,                     // Keyword::JSON
     DebeziumJson,             // Keyword::DEBEZIUM_JSON
@@ -138,26 +138,26 @@ pub enum SourceSchema {
     Bytes,
 }
 
-impl SourceSchema {
-    pub fn into_source_schema_v2(self) -> ConnectorSchema {
+impl LegacyRowFormat {
+    pub fn into_format_encode_v2(self) -> FormatEncodeOptions {
         let (format, row_encode) = match self {
-            SourceSchema::Protobuf(_) => (Format::Plain, Encode::Protobuf),
-            SourceSchema::Json => (Format::Plain, Encode::Json),
-            SourceSchema::DebeziumJson => (Format::Debezium, Encode::Json),
-            SourceSchema::DebeziumMongoJson => (Format::DebeziumMongo, Encode::Json),
-            SourceSchema::UpsertJson => (Format::Upsert, Encode::Json),
-            SourceSchema::Avro(_) => (Format::Plain, Encode::Avro),
-            SourceSchema::UpsertAvro(_) => (Format::Upsert, Encode::Avro),
-            SourceSchema::Maxwell => (Format::Maxwell, Encode::Json),
-            SourceSchema::CanalJson => (Format::Canal, Encode::Json),
-            SourceSchema::Csv(_) => (Format::Plain, Encode::Csv),
-            SourceSchema::DebeziumAvro(_) => (Format::Debezium, Encode::Avro),
-            SourceSchema::Bytes => (Format::Plain, Encode::Bytes),
-            SourceSchema::Native => (Format::Native, Encode::Native),
+            LegacyRowFormat::Protobuf(_) => (Format::Plain, Encode::Protobuf),
+            LegacyRowFormat::Json => (Format::Plain, Encode::Json),
+            LegacyRowFormat::DebeziumJson => (Format::Debezium, Encode::Json),
+            LegacyRowFormat::DebeziumMongoJson => (Format::DebeziumMongo, Encode::Json),
+            LegacyRowFormat::UpsertJson => (Format::Upsert, Encode::Json),
+            LegacyRowFormat::Avro(_) => (Format::Plain, Encode::Avro),
+            LegacyRowFormat::UpsertAvro(_) => (Format::Upsert, Encode::Avro),
+            LegacyRowFormat::Maxwell => (Format::Maxwell, Encode::Json),
+            LegacyRowFormat::CanalJson => (Format::Canal, Encode::Json),
+            LegacyRowFormat::Csv(_) => (Format::Plain, Encode::Csv),
+            LegacyRowFormat::DebeziumAvro(_) => (Format::Debezium, Encode::Avro),
+            LegacyRowFormat::Bytes => (Format::Plain, Encode::Bytes),
+            LegacyRowFormat::Native => (Format::Native, Encode::Native),
         };
 
         let row_options = match self {
-            SourceSchema::Protobuf(schema) => {
+            LegacyRowFormat::Protobuf(schema) => {
                 let mut options = vec![SqlOption {
                     name: ObjectName(vec![Ident {
                         value: "message".into(),
@@ -184,7 +184,7 @@ impl SourceSchema {
                 }
                 options
             }
-            SourceSchema::Avro(schema) | SourceSchema::UpsertAvro(schema) => {
+            LegacyRowFormat::Avro(schema) | LegacyRowFormat::UpsertAvro(schema) => {
                 if schema.use_schema_registry {
                     vec![SqlOption {
                         name: ObjectName(vec![Ident {
@@ -203,7 +203,7 @@ impl SourceSchema {
                     }]
                 }
             }
-            SourceSchema::DebeziumAvro(schema) => {
+            LegacyRowFormat::DebeziumAvro(schema) => {
                 vec![SqlOption {
                     name: ObjectName(vec![Ident {
                         value: "schema.registry".into(),
@@ -212,7 +212,7 @@ impl SourceSchema {
                     value: Value::SingleQuotedString(schema.row_schema_location.0),
                 }]
             }
-            SourceSchema::Csv(schema) => {
+            LegacyRowFormat::Csv(schema) => {
                 vec![
                     SqlOption {
                         name: ObjectName(vec![Ident {
@@ -239,7 +239,7 @@ impl SourceSchema {
             _ => vec![],
         };
 
-        ConnectorSchema {
+        FormatEncodeOptions {
             format,
             row_encode,
             row_options,
@@ -248,23 +248,27 @@ impl SourceSchema {
     }
 }
 
-impl fmt::Display for SourceSchema {
+impl fmt::Display for LegacyRowFormat {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "ROW FORMAT ")?;
         match self {
-            SourceSchema::Protobuf(protobuf_schema) => write!(f, "PROTOBUF {}", protobuf_schema),
-            SourceSchema::Json => write!(f, "JSON"),
-            SourceSchema::UpsertJson => write!(f, "UPSERT_JSON"),
-            SourceSchema::Maxwell => write!(f, "MAXWELL"),
-            SourceSchema::DebeziumJson => write!(f, "DEBEZIUM_JSON"),
-            SourceSchema::DebeziumMongoJson => write!(f, "DEBEZIUM_MONGO_JSON"),
-            SourceSchema::Avro(avro_schema) => write!(f, "AVRO {}", avro_schema),
-            SourceSchema::UpsertAvro(avro_schema) => write!(f, "UPSERT_AVRO {}", avro_schema),
-            SourceSchema::CanalJson => write!(f, "CANAL_JSON"),
-            SourceSchema::Csv(csv_info) => write!(f, "CSV {}", csv_info),
-            SourceSchema::Native => write!(f, "NATIVE"),
-            SourceSchema::DebeziumAvro(avro_schema) => write!(f, "DEBEZIUM_AVRO {}", avro_schema),
-            SourceSchema::Bytes => write!(f, "BYTES"),
+            LegacyRowFormat::Protobuf(protobuf_schema) => {
+                write!(f, "PROTOBUF {}", protobuf_schema)
+            }
+            LegacyRowFormat::Json => write!(f, "JSON"),
+            LegacyRowFormat::UpsertJson => write!(f, "UPSERT_JSON"),
+            LegacyRowFormat::Maxwell => write!(f, "MAXWELL"),
+            LegacyRowFormat::DebeziumJson => write!(f, "DEBEZIUM_JSON"),
+            LegacyRowFormat::DebeziumMongoJson => write!(f, "DEBEZIUM_MONGO_JSON"),
+            LegacyRowFormat::Avro(avro_schema) => write!(f, "AVRO {}", avro_schema),
+            LegacyRowFormat::UpsertAvro(avro_schema) => write!(f, "UPSERT_AVRO {}", avro_schema),
+            LegacyRowFormat::CanalJson => write!(f, "CANAL_JSON"),
+            LegacyRowFormat::Csv(csv_info) => write!(f, "CSV {}", csv_info),
+            LegacyRowFormat::Native => write!(f, "NATIVE"),
+            LegacyRowFormat::DebeziumAvro(avro_schema) => {
+                write!(f, "DEBEZIUM_AVRO {}", avro_schema)
+            }
+            LegacyRowFormat::Bytes => write!(f, "BYTES"),
         }
     }
 }
diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs
index daaf50f37e297..563dc66be4780 100644
--- a/src/sqlparser/src/ast/mod.rs
+++ b/src/sqlparser/src/ast/mod.rs
@@ -42,7 +42,7 @@ pub use self::ddl::{
     ReferentialAction, SourceWatermark, TableConstraint,
 };
 pub use self::legacy_source::{
-    get_delimiter, AvroSchema, CompatibleSourceSchema, DebeziumAvroSchema, ProtobufSchema,
+    get_delimiter, AvroSchema, CompatibleFormatEncode, DebeziumAvroSchema, ProtobufSchema,
 };
 pub use self::operator::{BinaryOperator, QualifiedOperator, UnaryOperator};
 pub use self::query::{
@@ -76,7 +76,7 @@ where
     sep: &'static str,
 }
 
-impl<'a, T> fmt::Display for DisplaySeparated<'a, T>
+impl<T> fmt::Display for DisplaySeparated<'_, T>
 where
     T: fmt::Display,
 {
@@ -1134,6 +1134,22 @@ impl fmt::Display for ExplainType {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+pub enum ExplainFormat {
+    Text,
+    Json,
+}
+
+impl fmt::Display for ExplainFormat {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            ExplainFormat::Text => f.write_str("TEXT"),
+            ExplainFormat::Json => f.write_str("JSON"),
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub struct ExplainOptions {
@@ -1143,6 +1159,8 @@ pub struct ExplainOptions {
     pub trace: bool,
     // explain's plan type
     pub explain_type: ExplainType,
+    // explain's plan format
+    pub explain_format: ExplainFormat,
 }
 
 impl Default for ExplainOptions {
@@ -1151,6 +1169,7 @@ impl Default for ExplainOptions {
             verbose: false,
             trace: false,
             explain_type: ExplainType::Physical,
+            explain_format: ExplainFormat::Text,
         }
     }
 }
@@ -1171,6 +1190,9 @@ impl fmt::Display for ExplainOptions {
             if self.explain_type == default.explain_type {
                 option_strs.push(self.explain_type.to_string());
             }
+            if self.explain_format == default.explain_format {
+                option_strs.push(self.explain_format.to_string());
+            }
             write!(f, "{}", option_strs.iter().format(","))
         }
     }
@@ -1264,8 +1286,8 @@ pub enum Statement {
         wildcard_idx: Option<usize>,
         constraints: Vec<TableConstraint>,
         with_options: Vec<SqlOption>,
-        /// Optional schema of the external source with which the table is created
-        source_schema: Option<CompatibleSourceSchema>,
+        /// `FORMAT ... ENCODE ...` for table with connector
+        format_encode: Option<CompatibleFormatEncode>,
         /// The watermark defined on source.
         source_watermarks: Vec<SourceWatermark>,
         /// Append only table.
@@ -1508,12 +1530,13 @@ pub enum Statement {
     CreateSchema {
         schema_name: ObjectName,
         if_not_exists: bool,
-        user_specified: Option<ObjectName>,
+        owner: Option<ObjectName>,
     },
     /// CREATE DATABASE
     CreateDatabase {
         db_name: ObjectName,
         if_not_exists: bool,
+        owner: Option<ObjectName>,
     },
     /// GRANT privileges ON objects TO grantees
     Grant {
@@ -1707,12 +1730,16 @@ impl fmt::Display for Statement {
             Statement::CreateDatabase {
                 db_name,
                 if_not_exists,
+                owner,
             } => {
                 write!(f, "CREATE DATABASE")?;
                 if *if_not_exists {
                     write!(f, " IF NOT EXISTS")?;
                 }
                 write!(f, " {}", db_name)?;
+                if let Some(owner) = owner {
+                    write!(f, " WITH OWNER = {}", owner)?;
+                }
                 Ok(())
             }
             Statement::CreateFunction {
@@ -1800,7 +1827,7 @@ impl fmt::Display for Statement {
                 or_replace,
                 if_not_exists,
                 temporary,
-                source_schema,
+                format_encode,
                 source_watermarks,
                 append_only,
                 on_conflict,
@@ -1847,8 +1874,8 @@ impl fmt::Display for Statement {
                 if !with_options.is_empty() {
                     write!(f, " WITH ({})", display_comma_separated(with_options))?;
                 }
-                if let Some(source_schema) = source_schema {
-                    write!(f, " {}", source_schema)?;
+                if let Some(format_encode) = format_encode {
+                    write!(f, " {}", format_encode)?;
                 }
                 if let Some(query) = query {
                     write!(f, " AS {}", query)?;
@@ -2036,7 +2063,7 @@ impl fmt::Display for Statement {
             Statement::CreateSchema {
                 schema_name,
                 if_not_exists,
-                user_specified,
+                owner,
             } => {
                 write!(
                     f,
@@ -2044,7 +2071,7 @@ impl fmt::Display for Statement {
                     if_not_exists = if *if_not_exists { "IF NOT EXISTS " } else { "" },
                     name = schema_name
                 )?;
-                if let Some(user) = user_specified {
+                if let Some(user) = owner {
                     write!(f, " AUTHORIZATION {}", user)?;
                 }
                 Ok(())
diff --git a/src/sqlparser/src/ast/query.rs b/src/sqlparser/src/ast/query.rs
index be03a5f1133ec..428fe4e4c5417 100644
--- a/src/sqlparser/src/ast/query.rs
+++ b/src/sqlparser/src/ast/query.rs
@@ -321,6 +321,7 @@ impl fmt::Display for With {
 }
 
 /// A single CTE (used after `WITH`): `alias [(col1, col2, ...)] AS ( query )`
+///
 /// The names in the column list before `AS`, when specified, replace the names
 /// of the columns returned by the query. The parser does not validate that the
 /// number of columns in the query matches the number of columns in the query.
@@ -335,8 +336,8 @@ impl fmt::Display for Cte {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match &self.cte_inner {
             CteInner::Query(query) => write!(f, "{} AS ({})", self.alias, query)?,
-            CteInner::ChangeLog(ident) => {
-                write!(f, "{} AS changelog from {}", self.alias, ident.value)?
+            CteInner::ChangeLog(obj_name) => {
+                write!(f, "{} AS changelog from {}", self.alias, obj_name)?
             }
         }
         Ok(())
@@ -347,7 +348,7 @@ impl fmt::Display for Cte {
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 pub enum CteInner {
     Query(Query),
-    ChangeLog(Ident),
+    ChangeLog(ObjectName),
 }
 
 /// One item of the comma-separated list following `SELECT`
@@ -464,9 +465,8 @@ impl fmt::Display for TableFactor {
         match self {
             TableFactor::Table { name, alias, as_of } => {
                 write!(f, "{}", name)?;
-                match as_of {
-                    Some(as_of) => write!(f, "{}", as_of)?,
-                    None => (),
+                if let Some(as_of) = as_of {
+                    write!(f, "{}", as_of)?
                 }
                 if let Some(alias) = alias {
                     write!(f, " AS {}", alias)?;
@@ -541,7 +541,7 @@ impl fmt::Display for Join {
         }
         fn suffix(constraint: &'_ JoinConstraint) -> impl fmt::Display + '_ {
             struct Suffix<'a>(&'a JoinConstraint);
-            impl<'a> fmt::Display for Suffix<'a> {
+            impl fmt::Display for Suffix<'_> {
                 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
                     match self.0 {
                         JoinConstraint::On(expr) => write!(f, " ON {}", expr),
diff --git a/src/sqlparser/src/ast/statement.rs b/src/sqlparser/src/ast/statement.rs
index a0f919091b56d..72680161defea 100644
--- a/src/sqlparser/src/ast/statement.rs
+++ b/src/sqlparser/src/ast/statement.rs
@@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize};
 use winnow::PResult;
 
 use super::ddl::SourceWatermark;
-use super::legacy_source::{parse_source_schema, CompatibleSourceSchema};
+use super::legacy_source::{parse_format_encode, CompatibleFormatEncode};
 use super::{EmitMode, Ident, ObjectType, Query, Value};
 use crate::ast::{
     display_comma_separated, display_separated, ColumnDef, ObjectName, SqlOption, TableConstraint,
@@ -76,7 +76,7 @@ macro_rules! impl_fmt_display {
 //     source_name: Ident,
 //     with_properties: AstOption<WithProperties>,
 //     [Keyword::ROW, Keyword::FORMAT],
-//     source_schema: SourceSchema,
+//     format_encode: SourceSchema,
 //     [Keyword::WATERMARK, Keyword::FOR] column [Keyword::AS] <expr>
 // });
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
@@ -90,7 +90,7 @@ pub struct CreateSourceStatement {
     pub constraints: Vec<TableConstraint>,
     pub source_name: ObjectName,
     pub with_properties: WithProperties,
-    pub source_schema: CompatibleSourceSchema,
+    pub format_encode: CompatibleFormatEncode,
     pub source_watermarks: Vec<SourceWatermark>,
     pub include_column_options: IncludeOption,
 }
@@ -222,7 +222,7 @@ impl Encode {
 /// `FORMAT ... ENCODE ... [(a=b, ...)] [KEY ENCODE ...]`
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-pub struct ConnectorSchema {
+pub struct FormatEncodeOptions {
     pub format: Format,
     pub row_encode: Encode,
     pub row_options: Vec<SqlOption>,
@@ -232,33 +232,33 @@ pub struct ConnectorSchema {
 
 impl Parser<'_> {
     /// Peek the next tokens to see if it is `FORMAT` or `ROW FORMAT` (for compatibility).
-    fn peek_source_schema_format(&mut self) -> bool {
+    fn peek_format_encode_format(&mut self) -> bool {
         (self.peek_nth_any_of_keywords(0, &[Keyword::ROW])
             && self.peek_nth_any_of_keywords(1, &[Keyword::FORMAT])) // ROW FORMAT
             || self.peek_nth_any_of_keywords(0, &[Keyword::FORMAT]) // FORMAT
     }
 
     /// Parse the source schema. The behavior depends on the `connector` type.
-    pub fn parse_source_schema_with_connector(
+    pub fn parse_format_encode_with_connector(
         &mut self,
         connector: &str,
         cdc_source_job: bool,
-    ) -> PResult<CompatibleSourceSchema> {
+    ) -> PResult<CompatibleFormatEncode> {
         // row format for cdc source must be debezium json
         // row format for nexmark source must be native
         // default row format for datagen source is native
         // FIXME: parse input `connector` to enum type instead using string here
         if connector.contains("-cdc") {
             let expected = if cdc_source_job {
-                ConnectorSchema::plain_json()
+                FormatEncodeOptions::plain_json()
             } else if connector.contains("mongodb") {
-                ConnectorSchema::debezium_mongo_json()
+                FormatEncodeOptions::debezium_mongo_json()
             } else {
-                ConnectorSchema::debezium_json()
+                FormatEncodeOptions::debezium_json()
             };
 
-            if self.peek_source_schema_format() {
-                let schema = parse_source_schema(self)?.into_v2();
+            if self.peek_format_encode_format() {
+                let schema = parse_format_encode(self)?.into_v2();
                 if schema != expected {
                     parser_err!(
                         "Row format for CDC connectors should be \
@@ -268,9 +268,9 @@ impl Parser<'_> {
             }
             Ok(expected.into())
         } else if connector.contains("nexmark") {
-            let expected = ConnectorSchema::native();
-            if self.peek_source_schema_format() {
-                let schema = parse_source_schema(self)?.into_v2();
+            let expected = FormatEncodeOptions::native();
+            if self.peek_format_encode_format() {
+                let schema = parse_format_encode(self)?.into_v2();
                 if schema != expected {
                     parser_err!(
                         "Row format for nexmark connectors should be \
@@ -280,15 +280,15 @@ impl Parser<'_> {
             }
             Ok(expected.into())
         } else if connector.contains("datagen") {
-            Ok(if self.peek_source_schema_format() {
-                parse_source_schema(self)?
+            Ok(if self.peek_format_encode_format() {
+                parse_format_encode(self)?
             } else {
-                ConnectorSchema::native().into()
+                FormatEncodeOptions::native().into()
             })
         } else if connector.contains("iceberg") {
-            let expected = ConnectorSchema::none();
-            if self.peek_source_schema_format() {
-                let schema = parse_source_schema(self)?.into_v2();
+            let expected = FormatEncodeOptions::none();
+            if self.peek_format_encode_format() {
+                let schema = parse_format_encode(self)?.into_v2();
                 if schema != expected {
                     parser_err!(
                         "Row format for iceberg connectors should be \
@@ -298,12 +298,12 @@ impl Parser<'_> {
             }
             Ok(expected.into())
         } else {
-            Ok(parse_source_schema(self)?)
+            Ok(parse_format_encode(self)?)
         }
     }
 
     /// Parse `FORMAT ... ENCODE ... (...)`.
-    pub fn parse_schema(&mut self) -> PResult<Option<ConnectorSchema>> {
+    pub fn parse_schema(&mut self) -> PResult<Option<FormatEncodeOptions>> {
         if !self.parse_keyword(Keyword::FORMAT) {
             return Ok(None);
         }
@@ -325,7 +325,7 @@ impl Parser<'_> {
             None
         };
 
-        Ok(Some(ConnectorSchema {
+        Ok(Some(FormatEncodeOptions {
             format,
             row_encode,
             row_options,
@@ -334,9 +334,9 @@ impl Parser<'_> {
     }
 }
 
-impl ConnectorSchema {
+impl FormatEncodeOptions {
     pub const fn plain_json() -> Self {
-        ConnectorSchema {
+        FormatEncodeOptions {
             format: Format::Plain,
             row_encode: Encode::Json,
             row_options: Vec::new(),
@@ -346,7 +346,7 @@ impl ConnectorSchema {
 
     /// Create a new source schema with `Debezium` format and `Json` encoding.
     pub const fn debezium_json() -> Self {
-        ConnectorSchema {
+        FormatEncodeOptions {
             format: Format::Debezium,
             row_encode: Encode::Json,
             row_options: Vec::new(),
@@ -355,7 +355,7 @@ impl ConnectorSchema {
     }
 
     pub const fn debezium_mongo_json() -> Self {
-        ConnectorSchema {
+        FormatEncodeOptions {
             format: Format::DebeziumMongo,
             row_encode: Encode::Json,
             row_options: Vec::new(),
@@ -365,7 +365,7 @@ impl ConnectorSchema {
 
     /// Create a new source schema with `Native` format and encoding.
     pub const fn native() -> Self {
-        ConnectorSchema {
+        FormatEncodeOptions {
             format: Format::Native,
             row_encode: Encode::Native,
             row_options: Vec::new(),
@@ -376,7 +376,7 @@ impl ConnectorSchema {
     /// Create a new source schema with `None` format and encoding.
     /// Used for self-explanatory source like iceberg.
     pub const fn none() -> Self {
-        ConnectorSchema {
+        FormatEncodeOptions {
             format: Format::None,
             row_encode: Encode::None,
             row_options: Vec::new(),
@@ -389,7 +389,7 @@ impl ConnectorSchema {
     }
 }
 
-impl fmt::Display for ConnectorSchema {
+impl fmt::Display for FormatEncodeOptions {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "FORMAT {} ENCODE {}", self.format, self.row_encode)?;
 
@@ -467,7 +467,7 @@ impl fmt::Display for CreateSourceStatement {
             v.push(format!("{}", item));
         }
         impl_fmt_display!(with_properties, v, self);
-        impl_fmt_display!(source_schema, v, self);
+        impl_fmt_display!(format_encode, v, self);
         v.iter().join(" ").fmt(f)
     }
 }
@@ -503,7 +503,7 @@ pub struct CreateSinkStatement {
     pub sink_from: CreateSink,
     pub columns: Vec<Ident>,
     pub emit_mode: Option<EmitMode>,
-    pub sink_schema: Option<ConnectorSchema>,
+    pub sink_schema: Option<FormatEncodeOptions>,
     pub into_table_name: Option<ObjectName>,
 }
 
diff --git a/src/sqlparser/src/ast/value.rs b/src/sqlparser/src/ast/value.rs
index 9cae715e09278..2bf8a6fdf3a02 100644
--- a/src/sqlparser/src/ast/value.rs
+++ b/src/sqlparser/src/ast/value.rs
@@ -181,7 +181,7 @@ impl fmt::Display for DateTimeField {
 
 pub struct EscapeSingleQuoteString<'a>(&'a str);
 
-impl<'a> fmt::Display for EscapeSingleQuoteString<'a> {
+impl fmt::Display for EscapeSingleQuoteString<'_> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         for c in self.0.chars() {
             if c == '\'' {
diff --git a/src/sqlparser/src/keywords.rs b/src/sqlparser/src/keywords.rs
index 8ec7191f749c2..93e47d7a6b11a 100644
--- a/src/sqlparser/src/keywords.rs
+++ b/src/sqlparser/src/keywords.rs
@@ -503,6 +503,7 @@ define_keywords!(
     SUCCEEDS,
     SUM,
     SUPERUSER,
+    SWAP,
     SYMMETRIC,
     SYNC,
     SYSTEM,
diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs
index b917f10ac458d..7d01a2b35cf67 100644
--- a/src/sqlparser/src/parser.rs
+++ b/src/sqlparser/src/parser.rs
@@ -2042,31 +2042,40 @@ impl Parser<'_> {
 
     pub fn parse_create_schema(&mut self) -> PResult<Statement> {
         let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
-        let (schema_name, user_specified) = if self.parse_keyword(Keyword::AUTHORIZATION) {
-            let user_specified = self.parse_object_name()?;
-            (user_specified.clone(), Some(user_specified))
+        let (schema_name, owner) = if self.parse_keyword(Keyword::AUTHORIZATION) {
+            let owner = self.parse_object_name()?;
+            (owner.clone(), Some(owner))
         } else {
             let schema_name = self.parse_object_name()?;
-            let user_specified = if self.parse_keyword(Keyword::AUTHORIZATION) {
+            let owner = if self.parse_keyword(Keyword::AUTHORIZATION) {
                 Some(self.parse_object_name()?)
             } else {
                 None
             };
-            (schema_name, user_specified)
+            (schema_name, owner)
         };
         Ok(Statement::CreateSchema {
             schema_name,
             if_not_exists,
-            user_specified,
+            owner,
         })
     }
 
     pub fn parse_create_database(&mut self) -> PResult<Statement> {
         let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
         let db_name = self.parse_object_name()?;
+        let _ = self.parse_keyword(Keyword::WITH);
+        let owner = if self.parse_keyword(Keyword::OWNER) {
+            let _ = self.consume_token(&Token::Eq);
+            Some(self.parse_object_name()?)
+        } else {
+            None
+        };
+
         Ok(Statement::CreateDatabase {
             db_name,
             if_not_exists,
+            owner,
         })
     }
 
@@ -2134,7 +2143,7 @@ impl Parser<'_> {
 
         // row format for nexmark source must be native
         // default row format for datagen source is native
-        let source_schema = self.parse_source_schema_with_connector(&connector, cdc_source_job)?;
+        let format_encode = self.parse_format_encode_with_connector(&connector, cdc_source_job)?;
 
         let stmt = CreateSourceStatement {
             temporary,
@@ -2144,7 +2153,7 @@ impl Parser<'_> {
             constraints,
             source_name,
             with_properties: WithProperties(with_options),
-            source_schema,
+            format_encode,
             source_watermarks,
             include_column_options: include_options,
         };
@@ -2576,8 +2585,8 @@ impl Parser<'_> {
             .find(|&opt| opt.name.real_value() == UPSTREAM_SOURCE_KEY);
         let connector = option.map(|opt| opt.value.to_string());
 
-        let source_schema = if let Some(connector) = connector {
-            Some(self.parse_source_schema_with_connector(&connector, false)?)
+        let format_encode = if let Some(connector) = connector {
+            Some(self.parse_format_encode_with_connector(&connector, false)?)
         } else {
             None // Table is NOT created with an external connector.
         };
@@ -2612,7 +2621,7 @@ impl Parser<'_> {
             with_options,
             or_replace,
             if_not_exists,
-            source_schema,
+            format_encode,
             source_watermarks,
             append_only,
             on_conflict,
@@ -3087,8 +3096,11 @@ impl Parser<'_> {
             self.expect_keyword(Keyword::TO)?;
             let schema_name = self.parse_object_name()?;
             AlterSchemaOperation::RenameSchema { schema_name }
+        } else if self.parse_keywords(&[Keyword::SWAP, Keyword::WITH]) {
+            let target_schema = self.parse_object_name()?;
+            AlterSchemaOperation::SwapRenameSchema { target_schema }
         } else {
-            return self.expected("RENAME OR OWNER TO after ALTER SCHEMA");
+            return self.expected("RENAME, OWNER TO, OR SWAP WITH after ALTER SCHEMA");
         };
 
         Ok(Statement::AlterSchema {
@@ -3207,8 +3219,12 @@ impl Parser<'_> {
             AlterTableOperation::AlterColumn { column_name, op }
         } else if self.parse_keywords(&[Keyword::REFRESH, Keyword::SCHEMA]) {
             AlterTableOperation::RefreshSchema
+        } else if self.parse_keywords(&[Keyword::SWAP, Keyword::WITH]) {
+            let target_table = self.parse_object_name()?;
+            AlterTableOperation::SwapRenameTable { target_table }
         } else {
-            return self.expected("ADD or RENAME or OWNER TO or SET or DROP after ALTER TABLE");
+            return self
+                .expected("ADD or RENAME or OWNER TO or SET or DROP or SWAP after ALTER TABLE");
         };
         Ok(Statement::AlterTable {
             name: table_name,
@@ -3313,6 +3329,9 @@ impl Parser<'_> {
             AlterViewOperation::ChangeOwner {
                 new_owner_name: owner_name,
             }
+        } else if self.parse_keywords(&[Keyword::SWAP, Keyword::WITH]) {
+            let target_view = self.parse_object_name()?;
+            AlterViewOperation::SwapRenameView { target_view }
         } else if self.parse_keyword(Keyword::SET) {
             if self.parse_keyword(Keyword::SCHEMA) {
                 let schema_name = self.parse_object_name()?;
@@ -3343,7 +3362,7 @@ impl Parser<'_> {
             }
         } else {
             return self.expected(&format!(
-                "RENAME or OWNER TO or SET after ALTER {}VIEW",
+                "RENAME or OWNER TO or SET or SWAP after ALTER {}VIEW",
                 if materialized { "MATERIALIZED " } else { "" }
             ));
         };
@@ -3392,6 +3411,9 @@ impl Parser<'_> {
             } else {
                 return self.expected("SCHEMA/PARALLELISM after SET");
             }
+        } else if self.parse_keywords(&[Keyword::SWAP, Keyword::WITH]) {
+            let target_sink = self.parse_object_name()?;
+            AlterSinkOperation::SwapRenameSink { target_sink }
         } else {
             return self.expected("RENAME or OWNER TO or SET after ALTER SINK");
         };
@@ -3425,8 +3447,13 @@ impl Parser<'_> {
             } else {
                 return self.expected("SCHEMA after SET");
             }
+        } else if self.parse_keywords(&[Keyword::SWAP, Keyword::WITH]) {
+            let target_subscription = self.parse_object_name()?;
+            AlterSubscriptionOperation::SwapRenameSubscription {
+                target_subscription,
+            }
         } else {
-            return self.expected("RENAME or OWNER TO or SET after ALTER SUBSCRIPTION");
+            return self.expected("RENAME or OWNER TO or SET or SWAP after ALTER SUBSCRIPTION");
         };
 
         Ok(Statement::AlterSubscription {
@@ -3466,13 +3493,16 @@ impl Parser<'_> {
                 return self.expected("SCHEMA after SET");
             }
         } else if self.peek_nth_any_of_keywords(0, &[Keyword::FORMAT]) {
-            let connector_schema = self.parse_schema()?.unwrap();
-            if connector_schema.key_encode.is_some() {
+            let format_encode = self.parse_schema()?.unwrap();
+            if format_encode.key_encode.is_some() {
                 parser_err!("key encode clause is not supported in source schema");
             }
-            AlterSourceOperation::FormatEncode { connector_schema }
+            AlterSourceOperation::FormatEncode { format_encode }
         } else if self.parse_keywords(&[Keyword::REFRESH, Keyword::SCHEMA]) {
             AlterSourceOperation::RefreshSchema
+        } else if self.parse_keywords(&[Keyword::SWAP, Keyword::WITH]) {
+            let target_source = self.parse_object_name()?;
+            AlterSourceOperation::SwapRenameSource { target_source }
         } else {
             return self.expected(
                 "RENAME, ADD COLUMN, OWNER TO, SET or SOURCE_RATE_LIMIT after ALTER SOURCE",
@@ -4006,6 +4036,7 @@ impl Parser<'_> {
             Keyword::LOGICAL,
             Keyword::PHYSICAL,
             Keyword::DISTSQL,
+            Keyword::FORMAT,
         ];
 
         let parse_explain_option = |parser: &mut Parser<'_>| -> PResult<()> {
@@ -4029,6 +4060,15 @@ impl Parser<'_> {
                 Keyword::LOGICAL => options.explain_type = ExplainType::Logical,
                 Keyword::PHYSICAL => options.explain_type = ExplainType::Physical,
                 Keyword::DISTSQL => options.explain_type = ExplainType::DistSql,
+                Keyword::FORMAT => {
+                    options.explain_format = {
+                        match parser.expect_one_of_keywords(&[Keyword::TEXT, Keyword::JSON])? {
+                            Keyword::TEXT => ExplainFormat::Text,
+                            Keyword::JSON => ExplainFormat::Json,
+                            _ => unreachable!("{}", keyword),
+                        }
+                    }
+                }
                 _ => unreachable!("{}", keyword),
             };
             Ok(())
@@ -4141,7 +4181,7 @@ impl Parser<'_> {
                 parser_err!("Expected 'changelog' but found '{}'", changelog);
             }
             self.expect_keyword(Keyword::FROM)?;
-            Ok(CteInner::ChangeLog(self.parse_identifier()?))
+            Ok(CteInner::ChangeLog(self.parse_object_name()?))
         }
     }
 
@@ -4959,6 +4999,7 @@ impl Parser<'_> {
                 Keyword::SCHEMA,
                 Keyword::TABLE,
                 Keyword::SOURCE,
+                Keyword::SINK,
             ]);
             let objects = self.parse_comma_separated(Parser::parse_object_name);
             match object_type {
@@ -4966,6 +5007,7 @@ impl Parser<'_> {
                 Some(Keyword::SCHEMA) => GrantObjects::Schemas(objects?),
                 Some(Keyword::SEQUENCE) => GrantObjects::Sequences(objects?),
                 Some(Keyword::SOURCE) => GrantObjects::Sources(objects?),
+                Some(Keyword::SINK) => GrantObjects::Sinks(objects?),
                 Some(Keyword::TABLE) | None => GrantObjects::Tables(objects?),
                 _ => unreachable!(),
             }
diff --git a/src/sqlparser/src/parser_v2/impl_.rs b/src/sqlparser/src/parser_v2/impl_.rs
index e4b6af2919745..328fccb200764 100644
--- a/src/sqlparser/src/parser_v2/impl_.rs
+++ b/src/sqlparser/src/parser_v2/impl_.rs
@@ -26,7 +26,7 @@ impl<'a> Offset<CheckpointWrapper<'a>> for CheckpointWrapper<'a> {
 }
 
 // Used for diagnostics with `--features winnow/debug`.
-impl<'a> std::fmt::Debug for Parser<'a> {
+impl std::fmt::Debug for Parser<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         if let Some(token) = self.0.first() {
             write!(f, "{}", token.token)?;
@@ -52,7 +52,7 @@ impl<'a> Offset<CheckpointWrapper<'a>> for Parser<'a> {
     }
 }
 
-impl<'a> SliceLen for Parser<'a> {
+impl SliceLen for Parser<'_> {
     #[inline(always)]
     fn slice_len(&self) -> usize {
         self.0.len()
@@ -135,7 +135,7 @@ impl<'a> Stream for Parser<'a> {
     }
 }
 
-impl<'a> UpdateSlice for Parser<'a> {
+impl UpdateSlice for Parser<'_> {
     #[inline(always)]
     fn update_slice(self, inner: Self::Slice) -> Self {
         Parser(self.0.update_slice(inner.0))
diff --git a/src/sqlparser/src/test_utils.rs b/src/sqlparser/src/test_utils.rs
index bfdf04dcc2d39..b6675928302cd 100644
--- a/src/sqlparser/src/test_utils.rs
+++ b/src/sqlparser/src/test_utils.rs
@@ -45,6 +45,7 @@ pub fn parse_sql_statements(sql: &str) -> Result<Vec<Statement>, ParserError> {
 }
 
 /// Ensures that `sql` parses as a single statement and returns it.
+///
 /// If non-empty `canonical` SQL representation is provided,
 /// additionally asserts that parsing `sql` results in the same parse
 /// tree as parsing `canonical`, and that serializing it back to string
diff --git a/src/sqlparser/tests/sqlparser_common.rs b/src/sqlparser/tests/sqlparser_common.rs
index 6fb7bbe5f6f51..c8f6fb41d32a9 100644
--- a/src/sqlparser/tests/sqlparser_common.rs
+++ b/src/sqlparser/tests/sqlparser_common.rs
@@ -1858,6 +1858,7 @@ fn parse_explain_analyze_with_simple_select() {
             trace: true,
             verbose: true,
             explain_type: ExplainType::DistSql,
+            explain_format: ExplainFormat::Text,
         },
     );
     run_explain_analyze(
@@ -1867,6 +1868,7 @@ fn parse_explain_analyze_with_simple_select() {
             trace: false,
             verbose: true,
             explain_type: ExplainType::DistSql,
+            explain_format: ExplainFormat::Text,
         },
     );
     run_explain_analyze(
@@ -1876,6 +1878,17 @@ fn parse_explain_analyze_with_simple_select() {
             trace: false,
             verbose: true,
             explain_type: ExplainType::DistSql,
+            explain_format: ExplainFormat::Text,
+        },
+    );
+    run_explain_analyze(
+        "EXPLAIN (LOGICAL, FORMAT JSON) SELECT sqrt(id) FROM foo",
+        false,
+        ExplainOptions {
+            trace: false,
+            verbose: false,
+            explain_type: ExplainType::Logical,
+            explain_format: ExplainFormat::Json,
         },
     );
 }
@@ -1893,9 +1906,15 @@ fn parse_explain_with_invalid_options() {
 
     let res = parse_sql_statements("EXPLAIN (VERBOSE, ) SELECT sqrt(id) FROM foo");
 
-    let err_msg =
-        "expected one of VERBOSE or TRACE or TYPE or LOGICAL or PHYSICAL or DISTSQL, found: )";
-    assert!(format!("{}", res.unwrap_err()).contains(err_msg));
+    let expected =
+        "expected one of VERBOSE or TRACE or TYPE or LOGICAL or PHYSICAL or DISTSQL or FORMAT, found: )";
+    let actual = res.unwrap_err().to_string();
+    assert!(
+        actual.contains(expected),
+        "expected: {:?}\nactual: {:?}",
+        expected,
+        actual
+    );
 }
 
 #[test]
@@ -2836,70 +2855,6 @@ fn parse_ctes() {
     }
 }
 
-#[test]
-fn parse_changelog_ctes() {
-    let cte_sqls = vec!["foo", "bar"];
-    let with = &format!(
-        "WITH a AS changelog from {}, b AS changelog from {} SELECT foo + bar FROM a, b",
-        cte_sqls[0], cte_sqls[1]
-    );
-
-    fn assert_changelog_ctes(expected: &[&str], sel: &Query) {
-        for (i, exp) in expected.iter().enumerate() {
-            let Cte { alias, cte_inner } = &sel.with.as_ref().unwrap().cte_tables[i];
-            if let CteInner::ChangeLog(from) = cte_inner {
-                assert_eq!(*exp, from.to_string());
-                assert_eq!(
-                    if i == 0 {
-                        Ident::new_unchecked("a")
-                    } else {
-                        Ident::new_unchecked("b")
-                    },
-                    alias.name
-                );
-                assert!(alias.columns.is_empty());
-            } else {
-                panic!("expected CteInner::ChangeLog")
-            }
-        }
-    }
-
-    // Top-level CTE
-    assert_changelog_ctes(&cte_sqls, &verified_query(with));
-    // CTE in a subquery
-    let sql = &format!("SELECT ({})", with);
-    let select = verified_only_select(sql);
-    match expr_from_projection(only(&select.projection)) {
-        Expr::Subquery(ref subquery) => {
-            assert_changelog_ctes(&cte_sqls, subquery.as_ref());
-        }
-        _ => panic!("expected subquery"),
-    }
-    // CTE in a derived table
-    let sql = &format!("SELECT * FROM ({})", with);
-    let select = verified_only_select(sql);
-    match only(select.from).relation {
-        TableFactor::Derived { subquery, .. } => {
-            assert_changelog_ctes(&cte_sqls, subquery.as_ref())
-        }
-        _ => panic!("expected derived table"),
-    }
-    // CTE in a view
-    let sql = &format!("CREATE VIEW v AS {}", with);
-    match verified_stmt(sql) {
-        Statement::CreateView { query, .. } => assert_changelog_ctes(&cte_sqls, &query),
-        _ => panic!("expected CREATE VIEW"),
-    }
-    // CTE in a CTE...
-    let sql = &format!("WITH outer_cte AS ({}) SELECT * FROM outer_cte", with);
-    let select = verified_query(sql);
-    if let CteInner::Query(query) = &only(&select.with.unwrap().cte_tables).cte_inner {
-        assert_changelog_ctes(&cte_sqls, query);
-    } else {
-        panic!("expected CteInner::Query")
-    }
-}
-
 #[test]
 fn parse_cte_renamed_columns() {
     let sql = "WITH cte (col1, col2) AS (SELECT foo, bar FROM baz) SELECT * FROM cte";
diff --git a/src/sqlparser/tests/testdata/create.yaml b/src/sqlparser/tests/testdata/create.yaml
index 9d0378f89c661..8390cc980cc25 100644
--- a/src/sqlparser/tests/testdata/create.yaml
+++ b/src/sqlparser/tests/testdata/create.yaml
@@ -1,16 +1,16 @@
 # This file is automatically generated by `src/sqlparser/tests/parser_test.rs`.
 - input: CREATE DATABASE t
   formatted_sql: CREATE DATABASE t
-  formatted_ast: 'CreateDatabase { db_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: false }'
+  formatted_ast: 'CreateDatabase { db_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: false, owner: None }'
 - input: CREATE DATABASE IF NOT EXISTS t
   formatted_sql: CREATE DATABASE IF NOT EXISTS t
-  formatted_ast: 'CreateDatabase { db_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: true }'
+  formatted_ast: 'CreateDatabase { db_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: true, owner: None }'
 - input: CREATE SCHEMA t
   formatted_sql: CREATE SCHEMA t
-  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: false, user_specified: None }'
+  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: false, owner: None }'
 - input: CREATE SCHEMA IF NOT EXISTS t
   formatted_sql: CREATE SCHEMA IF NOT EXISTS t
-  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: true, user_specified: None }'
+  formatted_ast: 'CreateSchema { schema_name: ObjectName([Ident { value: "t", quote_style: None }]), if_not_exists: true, owner: None }'
 - input: CREATE OR REPLACE TABLE t (a INT)
   formatted_sql: CREATE OR REPLACE TABLE t (a INT)
 - input: CREATE OR REPLACE TABLE t (a CHAR AS)
@@ -43,19 +43,19 @@
                                                                                                                      ^
 - input: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://')
   formatted_sql: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://')
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "location", quote_style: None }]), value: SingleQuotedString("file://") }], key_encode: None }), source_watermarks: [], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), format_encode: V2(FormatEncodeOptions { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "location", quote_style: None }]), value: SingleQuotedString("file://") }], key_encode: None }), source_watermarks: [], include_column_options: [] } }'
 - input: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
   formatted_sql: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }], key_encode: None }), source_watermarks: [], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), format_encode: V2(FormatEncodeOptions { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }], key_encode: None }), source_watermarks: [], include_column_options: [] } }'
 - input: CREATE SOURCE IF NOT EXISTS src (*, WATERMARK FOR event_time AS event_time - INTERVAL '60' SECOND) WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
   formatted_sql: CREATE SOURCE IF NOT EXISTS src (*, WATERMARK FOR event_time AS event_time - INTERVAL '60' SECOND) WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: Some(0), constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }], key_encode: None }), source_watermarks: [SourceWatermark { column: Ident { value: "event_time", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "event_time", quote_style: None }), op: Minus, right: Value(Interval { value: "60", leading_field: Some(Second), leading_precision: None, last_field: None, fractional_seconds_precision: None }) } }], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: Some(0), constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), format_encode: V2(FormatEncodeOptions { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }], key_encode: None }), source_watermarks: [SourceWatermark { column: Ident { value: "event_time", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "event_time", quote_style: None }), op: Minus, right: Value(Interval { value: "60", leading_field: Some(Second), leading_precision: None, last_field: None, fractional_seconds_precision: None }) } }], include_column_options: [] } }'
 - input: CREATE SOURCE IF NOT EXISTS src (PRIMARY KEY (event_id), WATERMARK FOR event_time AS event_time - INTERVAL '60' SECOND) WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
   formatted_sql: CREATE SOURCE IF NOT EXISTS src (PRIMARY KEY (event_id), WATERMARK FOR event_time AS event_time - INTERVAL '60' SECOND) WITH (kafka.topic = 'abc', kafka.brokers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://')
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: None, constraints: [Unique { name: None, columns: [Ident { value: "event_id", quote_style: None }], is_primary: true }], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }], key_encode: None }), source_watermarks: [SourceWatermark { column: Ident { value: "event_time", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "event_time", quote_style: None }), op: Minus, right: Value(Interval { value: "60", leading_field: Some(Second), leading_precision: None, last_field: None, fractional_seconds_precision: None }) } }], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: true, columns: [], wildcard_idx: None, constraints: [Unique { name: None, columns: [Ident { value: "event_id", quote_style: None }], is_primary: true }], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "brokers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), format_encode: V2(FormatEncodeOptions { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }], key_encode: None }), source_watermarks: [SourceWatermark { column: Ident { value: "event_time", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "event_time", quote_style: None }), op: Minus, right: Value(Interval { value: "60", leading_field: Some(Second), leading_precision: None, last_field: None, fractional_seconds_precision: None }) } }], include_column_options: [] } }'
 - input: CREATE SOURCE bid (auction INTEGER, bidder INTEGER, price INTEGER, WATERMARK FOR auction AS auction - 1, "date_time" TIMESTAMP) with (connector = 'nexmark', nexmark.table.type = 'Bid', nexmark.split.num = '12',  nexmark.min.event.gap.in.ns = '0')
   formatted_sql: CREATE SOURCE bid (auction INT, bidder INT, price INT, "date_time" TIMESTAMP, WATERMARK FOR auction AS auction - 1) WITH (connector = 'nexmark', nexmark.table.type = 'Bid', nexmark.split.num = '12', nexmark.min.event.gap.in.ns = '0') FORMAT NATIVE ENCODE NATIVE
-  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: false, columns: [ColumnDef { name: Ident { value: "auction", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "bidder", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "price", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "date_time", quote_style: Some(''"'') }, data_type: Some(Timestamp(false)), collation: None, options: [] }], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "bid", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "connector", quote_style: None }]), value: SingleQuotedString("nexmark") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "table", quote_style: None }, Ident { value: "type", quote_style: None }]), value: SingleQuotedString("Bid") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "split", quote_style: None }, Ident { value: "num", quote_style: None }]), value: SingleQuotedString("12") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "min", quote_style: None }, Ident { value: "event", quote_style: None }, Ident { value: "gap", quote_style: None }, Ident { value: "in", quote_style: None }, Ident { value: "ns", quote_style: None }]), value: SingleQuotedString("0") }]), source_schema: V2(ConnectorSchema { format: Native, row_encode: Native, row_options: [], key_encode: None }), source_watermarks: [SourceWatermark { column: Ident { value: "auction", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "auction", quote_style: None }), op: Minus, right: Value(Number("1")) } }], include_column_options: [] } }'
+  formatted_ast: 'CreateSource { stmt: CreateSourceStatement { temporary: false, if_not_exists: false, columns: [ColumnDef { name: Ident { value: "auction", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "bidder", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "price", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "date_time", quote_style: Some(''"'') }, data_type: Some(Timestamp(false)), collation: None, options: [] }], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "bid", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "connector", quote_style: None }]), value: SingleQuotedString("nexmark") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "table", quote_style: None }, Ident { value: "type", quote_style: None }]), value: SingleQuotedString("Bid") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "split", quote_style: None }, Ident { value: "num", quote_style: None }]), value: SingleQuotedString("12") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "min", quote_style: None }, Ident { value: "event", quote_style: None }, Ident { value: "gap", quote_style: None }, Ident { value: "in", quote_style: None }, Ident { value: "ns", quote_style: None }]), value: SingleQuotedString("0") }]), format_encode: V2(FormatEncodeOptions { format: Native, row_encode: Native, row_options: [], key_encode: None }), source_watermarks: [SourceWatermark { column: Ident { value: "auction", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "auction", quote_style: None }), op: Minus, right: Value(Number("1")) } }], include_column_options: [] } }'
 - input: |-
     CREATE SOURCE s
       (raw BYTEA)
diff --git a/src/sqlparser/tests/testdata/select.yaml b/src/sqlparser/tests/testdata/select.yaml
index 606b3d7a0c13f..72de88bd1dfb4 100644
--- a/src/sqlparser/tests/testdata/select.yaml
+++ b/src/sqlparser/tests/testdata/select.yaml
@@ -231,3 +231,9 @@
 - input: select date t; -- A column "date" aliased to "t"
   formatted_sql: SELECT date AS t
   formatted_ast: 'Query(Query { with: None, body: Select(Select { distinct: All, projection: [ExprWithAlias { expr: Identifier(Ident { value: "date", quote_style: None }), alias: Ident { value: "t", quote_style: None } }], from: [], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
+- input: |-
+    WITH a_log AS changelog from a,
+      "B_log" AS changelog from public."B"
+    SELECT a0 + b0 FROM a_log, "B_log"
+  formatted_sql: WITH a_log AS changelog from a, "B_log" AS changelog from public."B" SELECT a0 + b0 FROM a_log, "B_log"
+  formatted_ast: 'Query(Query { with: Some(With { recursive: false, cte_tables: [Cte { alias: TableAlias { name: Ident { value: "a_log", quote_style: None }, columns: [] }, cte_inner: ChangeLog(ObjectName([Ident { value: "a", quote_style: None }])) }, Cte { alias: TableAlias { name: Ident { value: "B_log", quote_style: Some(''"'') }, columns: [] }, cte_inner: ChangeLog(ObjectName([Ident { value: "public", quote_style: None }, Ident { value: "B", quote_style: Some(''"'') }])) }] }), body: Select(Select { distinct: All, projection: [UnnamedExpr(BinaryOp { left: Identifier(Ident { value: "a0", quote_style: None }), op: Plus, right: Identifier(Ident { value: "b0", quote_style: None }) })], from: [TableWithJoins { relation: Table { name: ObjectName([Ident { value: "a_log", quote_style: None }]), alias: None, as_of: None }, joins: [] }, TableWithJoins { relation: Table { name: ObjectName([Ident { value: "B_log", quote_style: Some(''"'') }]), alias: None, as_of: None }, joins: [] }], lateral_views: [], selection: None, group_by: [], having: None }), order_by: [], limit: None, offset: None, fetch: None })'
diff --git a/src/storage/backup/src/meta_snapshot_v1.rs b/src/storage/backup/src/meta_snapshot_v1.rs
index b873f863b6d05..c073dba1c2706 100644
--- a/src/storage/backup/src/meta_snapshot_v1.rs
+++ b/src/storage/backup/src/meta_snapshot_v1.rs
@@ -29,8 +29,7 @@ use risingwave_pb::user::UserInfo;
 use crate::error::{BackupError, BackupResult};
 use crate::meta_snapshot::{MetaSnapshot, Metadata};
 
-/// TODO: remove `ClusterMetadata` and even the trait, after applying model v2.
-
+// TODO: remove `ClusterMetadata` and even the trait, after applying model v2.
 pub type MetaSnapshotV1 = MetaSnapshot<ClusterMetadata>;
 
 impl Display for ClusterMetadata {
diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs
index 75ab989559d92..f09c5c3fa15b9 100644
--- a/src/storage/benches/bench_compactor.rs
+++ b/src/storage/benches/bench_compactor.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::ops::Range;
 use std::sync::Arc;
 
@@ -32,6 +33,7 @@ use risingwave_object_store::object::object_metrics::ObjectStoreMetrics;
 use risingwave_object_store::object::{InMemObjectStore, ObjectStore, ObjectStoreImpl};
 use risingwave_pb::hummock::compact_task::PbTaskType;
 use risingwave_pb::hummock::PbTableSchema;
+use risingwave_storage::compaction_catalog_manager::CompactionCatalogAgent;
 use risingwave_storage::hummock::compactor::compactor_runner::compact_and_build_sst;
 use risingwave_storage::hummock::compactor::{
     ConcatSstableIterator, DummyCompactionFilter, TaskConfig, TaskProgress,
@@ -133,8 +135,13 @@ async fn build_table(
             policy: CachePolicy::Fill(CacheContext::Default),
         },
     );
-    let mut builder =
-        SstableBuilder::<_, Xor16FilterBuilder>::for_test(sstable_object_id, writer, opt);
+    let table_id_to_vnode = HashMap::from_iter(vec![(0, VirtualNode::COUNT_FOR_TEST)]);
+    let mut builder = SstableBuilder::<_, Xor16FilterBuilder>::for_test(
+        sstable_object_id,
+        writer,
+        opt,
+        table_id_to_vnode,
+    );
     let value = b"1234567890123456789";
     let mut full_key = test_key_of(0, epoch, TableId::new(0));
     let table_key_len = full_key.user_key.table_key.len();
@@ -177,8 +184,14 @@ async fn build_table_2(
             policy: CachePolicy::Fill(CacheContext::Default),
         },
     );
-    let mut builder =
-        SstableBuilder::<_, Xor16FilterBuilder>::for_test(sstable_object_id, writer, opt);
+
+    let table_id_to_vnode = HashMap::from_iter(vec![(table_id, VirtualNode::COUNT_FOR_TEST)]);
+    let mut builder = SstableBuilder::<_, Xor16FilterBuilder>::for_test(
+        sstable_object_id,
+        writer,
+        opt,
+        table_id_to_vnode,
+    );
     let mut full_key = test_key_of(0, epoch, TableId::new(table_id));
     let table_key_len = full_key.user_key.table_key.len();
 
@@ -273,8 +286,11 @@ async fn compact<I: HummockIterator<Direction = Forward>>(
         bloom_false_positive: 0.001,
         ..Default::default()
     };
-    let mut builder =
-        CapacitySplitTableBuilder::for_test(LocalTableBuilderFactory::new(32, sstable_store, opt));
+    let compaction_catalog_agent_ref = CompactionCatalogAgent::for_test(vec![0]);
+    let mut builder = CapacitySplitTableBuilder::for_test(
+        LocalTableBuilderFactory::new(32, sstable_store, opt),
+        compaction_catalog_agent_ref,
+    );
 
     let task_config = task_config.unwrap_or_else(|| TaskConfig {
         key_range: KeyRange::inf(),
diff --git a/src/storage/benches/bench_multi_builder.rs b/src/storage/benches/bench_multi_builder.rs
index 2a7b379a55269..fc25674ad626b 100644
--- a/src/storage/benches/bench_multi_builder.rs
+++ b/src/storage/benches/bench_multi_builder.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::env;
 use std::ops::Range;
 use std::sync::atomic::AtomicU64;
@@ -24,11 +25,13 @@ use foyer::{Engine, HybridCacheBuilder};
 use rand::random;
 use risingwave_common::catalog::TableId;
 use risingwave_common::config::{MetricLevel, ObjectStoreConfig};
+use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::{FullKey, UserKey};
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_object_store::object::{
     InMemObjectStore, ObjectStore, ObjectStoreImpl, S3ObjectStore,
 };
+use risingwave_storage::compaction_catalog_manager::CompactionCatalogAgent;
 use risingwave_storage::hummock::iterator::{ConcatIterator, ConcatIteratorInner, HummockIterator};
 use risingwave_storage::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory};
 use risingwave_storage::hummock::value::HummockValue;
@@ -83,7 +86,11 @@ impl<F: SstableWriterFactory> TableBuilderFactory for LocalTableBuilderFactory<F
             .create_sst_writer(id, writer_options)
             .await
             .unwrap();
-        let builder = SstableBuilder::for_test(id, writer, self.options.clone());
+        let table_id_to_vnode = HashMap::from_iter(vec![(
+            TableId::default().into(),
+            VirtualNode::COUNT_FOR_TEST,
+        )]);
+        let builder = SstableBuilder::for_test(id, writer, self.options.clone(), table_id_to_vnode);
 
         Ok(builder)
     }
@@ -192,6 +199,8 @@ fn bench_builder(
 
     let sstable_store = runtime.block_on(async { generate_sstable_store(object_store).await });
 
+    let compaction_catalog_agent_ref = CompactionCatalogAgent::for_test(vec![0]);
+
     let mut group = c.benchmark_group("bench_multi_builder");
     group
         .sample_size(SAMPLE_COUNT)
@@ -205,6 +214,7 @@ fn bench_builder(
                         StreamingSstableWriterFactory::new(sstable_store.clone()),
                         get_builder_options(capacity_mb),
                     ),
+                    compaction_catalog_agent_ref.clone(),
                 ))
             })
         });
@@ -217,6 +227,7 @@ fn bench_builder(
                         BatchSstableWriterFactory::new(sstable_store.clone()),
                         get_builder_options(capacity_mb),
                     ),
+                    compaction_catalog_agent_ref.clone(),
                 ))
             })
         });
@@ -249,6 +260,8 @@ fn bench_table_scan(c: &mut Criterion) {
     let object_store = Arc::new(ObjectStoreImpl::InMem(store));
     let sstable_store = runtime.block_on(async { generate_sstable_store(object_store).await });
 
+    let compaction_catalog_agent_ref = CompactionCatalogAgent::for_test(vec![0]);
+
     let ssts = runtime.block_on(async {
         build_tables(CapacitySplitTableBuilder::for_test(
             LocalTableBuilderFactory::new(
@@ -256,6 +269,7 @@ fn bench_table_scan(c: &mut Criterion) {
                 BatchSstableWriterFactory::new(sstable_store.clone()),
                 get_builder_options(capacity_mb),
             ),
+            compaction_catalog_agent_ref.clone(),
         ))
         .await
     });
diff --git a/src/storage/compactor/src/compactor_observer/observer_manager.rs b/src/storage/compactor/src/compactor_observer/observer_manager.rs
index 97d78f6c2849f..0a04341ed0de9 100644
--- a/src/storage/compactor/src/compactor_observer/observer_manager.rs
+++ b/src/storage/compactor/src/compactor_observer/observer_manager.rs
@@ -12,21 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-use std::sync::Arc;
-
 use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef;
 use risingwave_common_service::ObserverState;
 use risingwave_pb::catalog::Table;
 use risingwave_pb::meta::relation::RelationInfo;
 use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::SubscribeResponse;
-use risingwave_storage::filter_key_extractor::{
-    FilterKeyExtractorImpl, FilterKeyExtractorManagerRef,
-};
+use risingwave_storage::compaction_catalog_manager::CompactionCatalogManagerRef;
 
 pub struct CompactorObserverNode {
-    filter_key_extractor_manager: FilterKeyExtractorManagerRef,
+    compaction_catalog_manager: CompactionCatalogManagerRef,
     system_params_manager: LocalSystemParamsManagerRef,
     version: u64,
 }
@@ -83,36 +78,30 @@ impl ObserverState for CompactorObserverNode {
 
 impl CompactorObserverNode {
     pub fn new(
-        filter_key_extractor_manager: FilterKeyExtractorManagerRef,
+        compaction_catalog_manager: CompactionCatalogManagerRef,
         system_params_manager: LocalSystemParamsManagerRef,
     ) -> Self {
         Self {
-            filter_key_extractor_manager,
+            compaction_catalog_manager,
             system_params_manager,
             version: 0,
         }
     }
 
     fn handle_catalog_snapshot(&mut self, tables: Vec<Table>) {
-        let all_filter_key_extractors: HashMap<u32, Arc<FilterKeyExtractorImpl>> = tables
-            .iter()
-            .map(|t| (t.id, Arc::new(FilterKeyExtractorImpl::from_table(t))))
-            .collect();
-        self.filter_key_extractor_manager
-            .sync(all_filter_key_extractors);
+        self.compaction_catalog_manager
+            .sync(tables.into_iter().map(|t| (t.id, t)).collect());
     }
 
     fn handle_catalog_notification(&mut self, operation: Operation, table_catalog: Table) {
         match operation {
             Operation::Add | Operation::Update => {
-                self.filter_key_extractor_manager.update(
-                    table_catalog.id,
-                    Arc::new(FilterKeyExtractorImpl::from_table(&table_catalog)),
-                );
+                self.compaction_catalog_manager
+                    .update(table_catalog.id, table_catalog);
             }
 
             Operation::Delete => {
-                self.filter_key_extractor_manager.remove(table_catalog.id);
+                self.compaction_catalog_manager.remove(table_catalog.id);
             }
 
             _ => panic!("receive an unsupported notify {:?}", operation),
diff --git a/src/storage/compactor/src/server.rs b/src/storage/compactor/src/server.rs
index e604cd34a2a1f..72ae1542f116a 100644
--- a/src/storage/compactor/src/server.rs
+++ b/src/storage/compactor/src/server.rs
@@ -36,8 +36,8 @@ use risingwave_pb::common::WorkerType;
 use risingwave_pb::compactor::compactor_service_server::CompactorServiceServer;
 use risingwave_pb::monitor_service::monitor_service_server::MonitorServiceServer;
 use risingwave_rpc_client::{GrpcCompactorProxyClient, MetaClient};
-use risingwave_storage::filter_key_extractor::{
-    FilterKeyExtractorManager, RemoteTableAccessor, RpcFilterKeyExtractorManager,
+use risingwave_storage::compaction_catalog_manager::{
+    CompactionCatalogManager, RemoteTableAccessor,
 };
 use risingwave_storage::hummock::compactor::{
     new_compaction_await_tree_reg_ref, CompactionAwaitTreeRegRef, CompactionExecutor,
@@ -212,12 +212,13 @@ pub async fn compactor_serve(
         compactor_metrics,
     ) = prepare_start_parameters(config.clone(), system_params_reader.clone()).await;
 
-    let filter_key_extractor_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new(
+    let compaction_catalog_manager_ref = Arc::new(CompactionCatalogManager::new(Box::new(
         RemoteTableAccessor::new(meta_client.clone()),
     )));
+
     let system_params_manager = Arc::new(LocalSystemParamsManager::new(system_params_reader));
     let compactor_observer_node = CompactorObserverNode::new(
-        filter_key_extractor_manager.clone(),
+        compaction_catalog_manager_ref.clone(),
         system_params_manager.clone(),
     );
     let observer_manager =
@@ -234,9 +235,6 @@ pub async fn compactor_serve(
         hummock_meta_client.clone(),
         storage_opts.sstable_id_remote_fetch_number,
     ));
-    let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-        filter_key_extractor_manager.clone(),
-    );
 
     let compaction_executor = Arc::new(CompactionExecutor::new(
         opts.compaction_worker_threads_number,
@@ -263,7 +261,7 @@ pub async fn compactor_serve(
             compactor_context.clone(),
             hummock_meta_client.clone(),
             sstable_object_id_manager.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_manager_ref,
         ),
     ];
 
diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
index a098efff62a2b..7a1f0851d3d4d 100644
--- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
+++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs
@@ -22,8 +22,7 @@ use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VnodeBitmapExt;
 use risingwave_pb::hummock::{
-    CompactionConfig, CompatibilityVersion, GroupConstruct, GroupMerge, PbLevelType,
-    StateTableInfo, StateTableInfoDelta,
+    CompactionConfig, CompatibilityVersion, PbLevelType, StateTableInfo, StateTableInfoDelta,
 };
 use tracing::warn;
 
@@ -36,91 +35,10 @@ use crate::level::{Level, Levels, OverlappingLevel};
 use crate::sstable_info::SstableInfo;
 use crate::table_watermark::{ReadTableWatermark, TableWatermarks};
 use crate::version::{
-    GroupDelta, GroupDeltas, HummockVersion, HummockVersionCommon, HummockVersionDelta,
-    HummockVersionStateTableInfo, IntraLevelDelta,
+    GroupDelta, GroupDeltaCommon, HummockVersion, HummockVersionCommon, HummockVersionDelta,
+    HummockVersionStateTableInfo, IntraLevelDelta, IntraLevelDeltaCommon,
 };
 use crate::{can_concat, CompactionGroupId, HummockSstableId, HummockSstableObjectId};
-
-pub struct GroupDeltasSummary {
-    pub delete_sst_levels: Vec<u32>,
-    pub delete_sst_ids_set: HashSet<u64>,
-    pub insert_sst_level_id: u32,
-    pub insert_sub_level_id: u64,
-    pub insert_table_infos: Vec<SstableInfo>,
-    pub group_construct: Option<GroupConstruct>,
-    pub group_destroy: Option<CompactionGroupId>,
-    pub new_vnode_partition_count: u32,
-    pub group_merge: Option<GroupMerge>,
-}
-
-pub fn summarize_group_deltas(
-    group_deltas: &GroupDeltas,
-    compaction_group_id: CompactionGroupId,
-) -> GroupDeltasSummary {
-    let mut delete_sst_levels = Vec::with_capacity(group_deltas.group_deltas.len());
-    let mut delete_sst_ids_set = HashSet::new();
-    let mut insert_sst_level_id = u32::MAX;
-    let mut insert_sub_level_id = u64::MAX;
-    let mut insert_table_infos = vec![];
-    let mut group_construct = None;
-    let mut group_destroy = None;
-    let mut new_vnode_partition_count = 0;
-    let mut group_merge = None;
-
-    for group_delta in &group_deltas.group_deltas {
-        match group_delta {
-            GroupDelta::IntraLevel(intra_level) => {
-                if !intra_level.removed_table_ids.is_empty() {
-                    delete_sst_levels.push(intra_level.level_idx);
-                    delete_sst_ids_set.extend(intra_level.removed_table_ids.iter().clone());
-                }
-                if !intra_level.inserted_table_infos.is_empty() {
-                    insert_sst_level_id = intra_level.level_idx;
-                    insert_sub_level_id = intra_level.l0_sub_level_id;
-                    insert_table_infos.extend(intra_level.inserted_table_infos.iter().cloned());
-                }
-                new_vnode_partition_count = intra_level.vnode_partition_count;
-            }
-            GroupDelta::GroupConstruct(construct_delta) => {
-                assert!(group_construct.is_none());
-                group_construct = Some(construct_delta.clone());
-            }
-            GroupDelta::GroupDestroy(_) => {
-                assert!(group_destroy.is_none());
-                group_destroy = Some(compaction_group_id);
-            }
-            GroupDelta::GroupMerge(merge_delta) => {
-                assert!(group_merge.is_none());
-                group_merge = Some(*merge_delta);
-                group_destroy = Some(merge_delta.right_group_id);
-            }
-        }
-    }
-
-    delete_sst_levels.sort();
-    delete_sst_levels.dedup();
-
-    GroupDeltasSummary {
-        delete_sst_levels,
-        delete_sst_ids_set,
-        insert_sst_level_id,
-        insert_sub_level_id,
-        insert_table_infos,
-        group_construct,
-        group_destroy,
-        new_vnode_partition_count,
-        group_merge,
-    }
-}
-
-#[derive(Clone, Default)]
-pub struct TableGroupInfo {
-    pub group_id: CompactionGroupId,
-    pub group_size: u64,
-    pub table_statistic: HashMap<StateTableId, u64>,
-    pub split_by_table: bool,
-}
-
 #[derive(Debug, Clone, Default)]
 pub struct SstDeltaInfo {
     pub insert_sst_level: u32,
@@ -198,7 +116,7 @@ impl HummockVersion {
     pub fn get_sst_infos_from_groups<'a>(
         &'a self,
         select_group: &'a HashSet<CompactionGroupId>,
-    ) -> impl Iterator<Item = &SstableInfo> + 'a {
+    ) -> impl Iterator<Item = &'a SstableInfo> + 'a {
         self.levels
             .iter()
             .filter_map(|(cg_id, level)| {
@@ -335,6 +253,8 @@ impl HummockVersion {
                 let l0 = &parent_levels.l0;
                 let mut split_count = 0;
                 for sub_level in &l0.sub_levels {
+                    assert!(!sub_level.table_infos.is_empty());
+
                     if sub_level.level_type == PbLevelType::Overlapping {
                         // TODO: use table_id / vnode / key_range filter
                         split_count += sub_level
@@ -412,7 +332,7 @@ impl HummockVersion {
         let [parent_levels, cur_levels] = self
             .levels
             .get_many_mut([&parent_group_id, &group_id])
-            .unwrap();
+            .map(|res| res.unwrap());
         let l0 = &mut parent_levels.l0;
         {
             for sub_level in &mut l0.sub_levels {
@@ -448,6 +368,8 @@ impl HummockVersion {
                     }
                 }
             }
+
+            l0.sub_levels.retain(|level| !level.table_infos.is_empty());
         }
         for (idx, level) in parent_levels.levels.iter_mut().enumerate() {
             let insert_table_infos =
@@ -475,6 +397,17 @@ impl HummockVersion {
                     level.uncompressed_file_size -= sst_info.uncompressed_file_size;
                 });
         }
+
+        assert!(parent_levels
+            .l0
+            .sub_levels
+            .iter()
+            .all(|level| !level.table_infos.is_empty()));
+        assert!(cur_levels
+            .l0
+            .sub_levels
+            .iter()
+            .all(|level| !level.table_infos.is_empty()));
     }
 
     pub fn build_sst_delta_infos(&self, version_delta: &HummockVersionDelta) -> Vec<SstDeltaInfo> {
@@ -493,11 +426,12 @@ impl HummockVersion {
             let mut removed_ssts: BTreeMap<u32, BTreeSet<u64>> = BTreeMap::new();
 
             // Build only if all deltas are intra level deltas.
-            if !group_deltas
-                .group_deltas
-                .iter()
-                .all(|delta| matches!(delta, GroupDelta::IntraLevel(_)))
-            {
+            if !group_deltas.group_deltas.iter().all(|delta| {
+                matches!(
+                    delta,
+                    GroupDelta::IntraLevel(_) | GroupDelta::NewL0SubLevel(_)
+                )
+            }) {
                 continue;
             }
 
@@ -505,24 +439,36 @@ impl HummockVersion {
             // current `hummock::manager::gen_version_delta` implementation. Better refactor the
             // struct to reduce conventions.
             for group_delta in &group_deltas.group_deltas {
-                if let GroupDelta::IntraLevel(intra_level) = group_delta {
-                    if !intra_level.inserted_table_infos.is_empty() {
-                        info.insert_sst_level = intra_level.level_idx;
-                        info.insert_sst_infos
-                            .extend(intra_level.inserted_table_infos.iter().cloned());
+                match group_delta {
+                    GroupDeltaCommon::NewL0SubLevel(inserted_table_infos) => {
+                        if !inserted_table_infos.is_empty() {
+                            info.insert_sst_level = 0;
+                            info.insert_sst_infos
+                                .extend(inserted_table_infos.iter().cloned());
+                        }
                     }
-                    if !intra_level.removed_table_ids.is_empty() {
-                        for id in &intra_level.removed_table_ids {
-                            if intra_level.level_idx == 0 {
-                                removed_l0_ssts.insert(*id);
-                            } else {
-                                removed_ssts
-                                    .entry(intra_level.level_idx)
-                                    .or_default()
-                                    .insert(*id);
+                    GroupDeltaCommon::IntraLevel(intra_level) => {
+                        if !intra_level.inserted_table_infos.is_empty() {
+                            info.insert_sst_level = intra_level.level_idx;
+                            info.insert_sst_infos
+                                .extend(intra_level.inserted_table_infos.iter().cloned());
+                        }
+                        if !intra_level.removed_table_ids.is_empty() {
+                            for id in &intra_level.removed_table_ids {
+                                if intra_level.level_idx == 0 {
+                                    removed_l0_ssts.insert(*id);
+                                } else {
+                                    removed_ssts
+                                        .entry(intra_level.level_idx)
+                                        .or_default()
+                                        .insert(*id);
+                                }
                             }
                         }
                     }
+                    GroupDeltaCommon::GroupConstruct(_)
+                    | GroupDeltaCommon::GroupDestroy(_)
+                    | GroupDeltaCommon::GroupMerge(_) => {}
                 }
             }
 
@@ -587,97 +533,129 @@ impl HummockVersion {
 
         // apply to `levels`, which is different compaction groups
         for (compaction_group_id, group_deltas) in &version_delta.group_deltas {
-            let summary = summarize_group_deltas(group_deltas, *compaction_group_id);
-            if let Some(group_construct) = &summary.group_construct {
-                let mut new_levels = build_initial_compaction_group_levels(
-                    *compaction_group_id,
-                    group_construct.get_group_config().unwrap(),
-                );
-                let parent_group_id = group_construct.parent_group_id;
-                new_levels.parent_group_id = parent_group_id;
-                #[expect(deprecated)]
-                // for backward-compatibility of previous hummock version delta
-                new_levels
-                    .member_table_ids
-                    .clone_from(&group_construct.table_ids);
-                self.levels.insert(*compaction_group_id, new_levels);
-                let member_table_ids =
-                    if group_construct.version >= CompatibilityVersion::NoMemberTableIds as _ {
-                        self.state_table_info
-                            .compaction_group_member_table_ids(*compaction_group_id)
-                            .iter()
-                            .map(|table_id| table_id.table_id)
-                            .collect()
-                    } else {
+            let mut is_applied_l0_compact = false;
+            for group_delta in &group_deltas.group_deltas {
+                match group_delta {
+                    GroupDeltaCommon::GroupConstruct(group_construct) => {
+                        let mut new_levels = build_initial_compaction_group_levels(
+                            *compaction_group_id,
+                            group_construct.get_group_config().unwrap(),
+                        );
+                        let parent_group_id = group_construct.parent_group_id;
+                        new_levels.parent_group_id = parent_group_id;
                         #[expect(deprecated)]
                         // for backward-compatibility of previous hummock version delta
-                        BTreeSet::from_iter(group_construct.table_ids.clone())
-                    };
-
-                if group_construct.version >= CompatibilityVersion::SplitGroupByTableId as _ {
-                    let split_key = if group_construct.split_key.is_some() {
-                        Some(Bytes::from(group_construct.split_key.clone().unwrap()))
-                    } else {
-                        None
-                    };
-                    self.init_with_parent_group_v2(
-                        parent_group_id,
-                        *compaction_group_id,
-                        group_construct.get_new_sst_start_id(),
-                        split_key.clone(),
-                    );
-                } else {
-                    // for backward-compatibility of previous hummock version delta
-                    self.init_with_parent_group(
-                        parent_group_id,
-                        *compaction_group_id,
-                        member_table_ids,
-                        group_construct.get_new_sst_start_id(),
-                    );
-                }
-            } else if let Some(group_merge) = &summary.group_merge {
-                tracing::info!(
-                    "group_merge left {:?} right {:?}",
-                    group_merge.left_group_id,
-                    group_merge.right_group_id
-                );
-                self.merge_compaction_group(group_merge.left_group_id, group_merge.right_group_id)
-            }
-            let group_destroy = summary.group_destroy;
-            let levels = self.levels.get_mut(compaction_group_id).unwrap_or_else(|| {
-                panic!("compaction group {} does not exist", compaction_group_id)
-            });
+                        new_levels
+                            .member_table_ids
+                            .clone_from(&group_construct.table_ids);
+                        self.levels.insert(*compaction_group_id, new_levels);
+                        let member_table_ids = if group_construct.version
+                            >= CompatibilityVersion::NoMemberTableIds as _
+                        {
+                            self.state_table_info
+                                .compaction_group_member_table_ids(*compaction_group_id)
+                                .iter()
+                                .map(|table_id| table_id.table_id)
+                                .collect()
+                        } else {
+                            #[expect(deprecated)]
+                            // for backward-compatibility of previous hummock version delta
+                            BTreeSet::from_iter(group_construct.table_ids.clone())
+                        };
+
+                        if group_construct.version >= CompatibilityVersion::SplitGroupByTableId as _
+                        {
+                            let split_key = if group_construct.split_key.is_some() {
+                                Some(Bytes::from(group_construct.split_key.clone().unwrap()))
+                            } else {
+                                None
+                            };
+                            self.init_with_parent_group_v2(
+                                parent_group_id,
+                                *compaction_group_id,
+                                group_construct.get_new_sst_start_id(),
+                                split_key.clone(),
+                            );
+                        } else {
+                            // for backward-compatibility of previous hummock version delta
+                            self.init_with_parent_group(
+                                parent_group_id,
+                                *compaction_group_id,
+                                member_table_ids,
+                                group_construct.get_new_sst_start_id(),
+                            );
+                        }
+                    }
+                    GroupDeltaCommon::GroupMerge(group_merge) => {
+                        tracing::info!(
+                            "group_merge left {:?} right {:?}",
+                            group_merge.left_group_id,
+                            group_merge.right_group_id
+                        );
+                        self.merge_compaction_group(
+                            group_merge.left_group_id,
+                            group_merge.right_group_id,
+                        )
+                    }
+                    GroupDeltaCommon::IntraLevel(level_delta) => {
+                        let levels =
+                            self.levels.get_mut(compaction_group_id).unwrap_or_else(|| {
+                                panic!("compaction group {} does not exist", compaction_group_id)
+                            });
+                        if is_commit_epoch {
+                            assert!(
+                                level_delta.removed_table_ids.is_empty(),
+                                "no sst should be deleted when committing an epoch"
+                            );
 
-            if is_commit_epoch {
-                let GroupDeltasSummary {
-                    delete_sst_levels,
-                    delete_sst_ids_set,
-                    ..
-                } = summary;
+                            let IntraLevelDelta {
+                                level_idx,
+                                l0_sub_level_id,
+                                inserted_table_infos,
+                                ..
+                            } = level_delta;
+                            {
+                                assert_eq!(
+                                    *level_idx, 0,
+                                    "we should only add to L0 when we commit an epoch."
+                                );
+                                if !inserted_table_infos.is_empty() {
+                                    insert_new_sub_level(
+                                        &mut levels.l0,
+                                        *l0_sub_level_id,
+                                        PbLevelType::Overlapping,
+                                        inserted_table_infos.clone(),
+                                        None,
+                                    );
+                                }
+                            }
+                        } else {
+                            // The delta is caused by compaction.
+                            levels.apply_compact_ssts(
+                                level_delta,
+                                self.state_table_info
+                                    .compaction_group_member_table_ids(*compaction_group_id),
+                            );
+                            if level_delta.level_idx == 0 {
+                                is_applied_l0_compact = true;
+                            }
+                        }
+                    }
+                    GroupDeltaCommon::NewL0SubLevel(inserted_table_infos) => {
+                        let levels =
+                            self.levels.get_mut(compaction_group_id).unwrap_or_else(|| {
+                                panic!("compaction group {} does not exist", compaction_group_id)
+                            });
+                        assert!(is_commit_epoch);
 
-                assert!(
-                    delete_sst_levels.is_empty() && delete_sst_ids_set.is_empty()
-                        || group_destroy.is_some(),
-                    "no sst should be deleted when committing an epoch"
-                );
-                let mut next_l0_sub_level_id = levels
-                    .l0
-                    .sub_levels
-                    .last()
-                    .map(|level| level.sub_level_id + 1)
-                    .unwrap_or(1);
-                for group_delta in &group_deltas.group_deltas {
-                    if let GroupDelta::IntraLevel(IntraLevelDelta {
-                        level_idx,
-                        inserted_table_infos,
-                        ..
-                    }) = group_delta
-                    {
-                        assert_eq!(
-                            *level_idx, 0,
-                            "we should only add to L0 when we commit an epoch."
-                        );
                         if !inserted_table_infos.is_empty() {
+                            let next_l0_sub_level_id = levels
+                                .l0
+                                .sub_levels
+                                .last()
+                                .map(|level| level.sub_level_id + 1)
+                                .unwrap_or(1);
+
                             insert_new_sub_level(
                                 &mut levels.l0,
                                 next_l0_sub_level_id,
@@ -685,20 +663,16 @@ impl HummockVersion {
                                 inserted_table_infos.clone(),
                                 None,
                             );
-                            next_l0_sub_level_id += 1;
                         }
                     }
+                    GroupDeltaCommon::GroupDestroy(_) => {
+                        self.levels.remove(compaction_group_id);
+                    }
                 }
-            } else {
-                // The delta is caused by compaction.
-                levels.apply_compact_ssts(
-                    summary,
-                    self.state_table_info
-                        .compaction_group_member_table_ids(*compaction_group_id),
-                );
             }
-            if let Some(destroy_group_id) = &group_destroy {
-                self.levels.remove(destroy_group_id);
+            if is_applied_l0_compact && let Some(levels) = self.levels.get_mut(compaction_group_id)
+            {
+                levels.post_apply_l0_compact();
             }
         }
         self.id = version_delta.id;
@@ -921,7 +895,8 @@ impl HummockVersionCommon<SstableInfo> {
         let [parent_levels, cur_levels] = self
             .levels
             .get_many_mut([&parent_group_id, &group_id])
-            .unwrap();
+            .map(|res| res.unwrap());
+
         let l0 = &mut parent_levels.l0;
         {
             for sub_level in &mut l0.sub_levels {
@@ -966,6 +941,8 @@ impl HummockVersionCommon<SstableInfo> {
                     }
                 }
             }
+
+            l0.sub_levels.retain(|level| !level.table_infos.is_empty());
         }
 
         for (idx, level) in parent_levels.levels.iter_mut().enumerate() {
@@ -1002,57 +979,67 @@ impl HummockVersionCommon<SstableInfo> {
                     level.uncompressed_file_size -= sst_info.uncompressed_file_size;
                 });
         }
+
+        assert!(parent_levels
+            .l0
+            .sub_levels
+            .iter()
+            .all(|level| !level.table_infos.is_empty()));
+        assert!(cur_levels
+            .l0
+            .sub_levels
+            .iter()
+            .all(|level| !level.table_infos.is_empty()));
     }
 }
 
-#[easy_ext::ext(HummockLevelsExt)]
 impl Levels {
-    pub fn apply_compact_ssts(
+    pub(crate) fn apply_compact_ssts(
         &mut self,
-        summary: GroupDeltasSummary,
+        level_delta: &IntraLevelDeltaCommon<SstableInfo>,
         member_table_ids: &BTreeSet<TableId>,
     ) {
-        let GroupDeltasSummary {
-            delete_sst_levels,
-            delete_sst_ids_set,
-            insert_sst_level_id,
-            insert_sub_level_id,
-            insert_table_infos,
-            new_vnode_partition_count,
+        let IntraLevelDeltaCommon {
+            level_idx,
+            l0_sub_level_id,
+            inserted_table_infos: insert_table_infos,
+            vnode_partition_count,
+            removed_table_ids: delete_sst_ids_set,
             ..
-        } = summary;
+        } = level_delta;
+        let new_vnode_partition_count = *vnode_partition_count;
 
-        if !self.check_sst_ids_exist(&delete_sst_levels, delete_sst_ids_set.clone()) {
+        if !self.check_sst_ids_exist(&[*level_idx], delete_sst_ids_set.clone()) {
             warn!(
                 "This VersionDelta may be committed by an expired compact task. Please check it. \n
-                    delete_sst_levels: {:?}\n,
-                    delete_sst_ids_set: {:?}\n,
                     insert_sst_level_id: {}\n,
                     insert_sub_level_id: {}\n,
-                    insert_table_infos: {:?}\n",
-                delete_sst_levels,
-                delete_sst_ids_set,
-                insert_sst_level_id,
-                insert_sub_level_id,
+                    insert_table_infos: {:?}\n,
+                    delete_sst_ids_set: {:?}\n",
+                level_idx,
+                l0_sub_level_id,
                 insert_table_infos
                     .iter()
                     .map(|sst| (sst.sst_id, sst.object_id))
-                    .collect_vec()
+                    .collect_vec(),
+                delete_sst_ids_set,
             );
             return;
         }
-        for level_idx in &delete_sst_levels {
+        if !delete_sst_ids_set.is_empty() {
             if *level_idx == 0 {
                 for level in &mut self.l0.sub_levels {
-                    level_delete_ssts(level, &delete_sst_ids_set);
+                    level_delete_ssts(level, delete_sst_ids_set);
                 }
             } else {
                 let idx = *level_idx as usize - 1;
-                level_delete_ssts(&mut self.levels[idx], &delete_sst_ids_set);
+                level_delete_ssts(&mut self.levels[idx], delete_sst_ids_set);
             }
         }
 
         if !insert_table_infos.is_empty() {
+            let insert_sst_level_id = *level_idx;
+            let insert_sub_level_id = *l0_sub_level_id;
             if insert_sst_level_id == 0 {
                 let l0 = &mut self.l0;
                 let index = l0
@@ -1093,7 +1080,10 @@ impl Levels {
                 level_insert_ssts(&mut self.levels[idx], insert_table_infos);
             }
         }
-        if delete_sst_levels.iter().any(|level_id| *level_id == 0) {
+    }
+
+    pub(crate) fn post_apply_l0_compact(&mut self) {
+        {
             self.l0
                 .sub_levels
                 .retain(|level| !level.table_infos.is_empty());
@@ -1358,7 +1348,7 @@ fn level_delete_ssts(
     original_len != operand.table_infos.len()
 }
 
-fn level_insert_ssts(operand: &mut Level, insert_table_infos: Vec<SstableInfo>) {
+fn level_insert_ssts(operand: &mut Level, insert_table_infos: &Vec<SstableInfo>) {
     operand.total_file_size += insert_table_infos
         .iter()
         .map(|sst| sst.sst_size)
@@ -1367,7 +1357,9 @@ fn level_insert_ssts(operand: &mut Level, insert_table_infos: Vec<SstableInfo>)
         .iter()
         .map(|sst| sst.uncompressed_file_size)
         .sum::<u64>();
-    operand.table_infos.extend(insert_table_infos);
+    operand
+        .table_infos
+        .extend(insert_table_infos.iter().cloned());
     operand
         .table_infos
         .sort_by(|sst1, sst2| sst1.key_range.cmp(&sst2.key_range));
@@ -1501,7 +1493,7 @@ pub fn validate_version(version: &HummockVersion) -> Vec<String> {
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashMap;
+    use std::collections::{HashMap, HashSet};
 
     use bytes::Bytes;
     use risingwave_common::catalog::TableId;
@@ -1655,7 +1647,7 @@ mod tests {
                         group_deltas: vec![GroupDelta::IntraLevel(IntraLevelDelta::new(
                             1,
                             0,
-                            vec![],
+                            HashSet::new(),
                             vec![SstableInfo {
                                 object_id: 1,
                                 sst_id: 1,
diff --git a/src/storage/hummock_sdk/src/compaction_group/mod.rs b/src/storage/hummock_sdk/src/compaction_group/mod.rs
index 0bf3fcea1c4e7..dbe58018f15e3 100644
--- a/src/storage/hummock_sdk/src/compaction_group/mod.rs
+++ b/src/storage/hummock_sdk/src/compaction_group/mod.rs
@@ -44,6 +44,10 @@ impl From<StaticCompactionGroupId> for CompactionGroupId {
     }
 }
 
+/// The split will follow the following rules:
+/// 1. Ssts with `split_key` will be split into two separate ssts and their `key_range` will be changed `sst_1`: [`sst.key_range.right`, `split_key`) `sst_2`: [`split_key`, `sst.key_range.right`].
+/// 2. Currently only `vnode` 0 and `vnode` max is supported.
+/// 3. Due to the above rule, `vnode` max will be rewritten as `table_id` + 1, vnode 0
 pub mod group_split {
     use std::cmp::Ordering;
     use std::collections::BTreeSet;
@@ -61,11 +65,6 @@ pub mod group_split {
     use crate::sstable_info::SstableInfo;
     use crate::{can_concat, HummockEpoch, KeyComparator};
 
-    /// The split will follow the following rules:
-    /// 1. Ssts with `split_key` will be split into two separate ssts and their `key_range` will be changed `sst_1`: [`sst.key_range.right`, `split_key`) `sst_2`: [`split_key`, `sst.key_range.right`].
-    /// 2. Currently only `vnode` 0 and `vnode` max is supported.
-    /// 3. Due to the above rule, `vnode` max will be rewritten as `table_id` + 1, vnode 0
-
     // By default, the split key is constructed with vnode = 0 and epoch = MAX, so that we can split table_id to the right group
     pub fn build_split_key(table_id: StateTableId, vnode: VirtualNode) -> Bytes {
         build_split_full_key(table_id, vnode).encode().into()
diff --git a/src/storage/hummock_sdk/src/key.rs b/src/storage/hummock_sdk/src/key.rs
index 9685b0cff3ecc..3318756d66673 100644
--- a/src/storage/hummock_sdk/src/key.rs
+++ b/src/storage/hummock_sdk/src/key.rs
@@ -292,7 +292,6 @@ pub fn prev_epoch(epoch: &[u8]) -> Vec<u8> {
 /// compute the next full key of the given full key
 ///
 /// if the `user_key` has no successor key, the result will be a empty vec
-
 pub fn next_full_key(full_key: &[u8]) -> Vec<u8> {
     let (user_key, epoch) = split_key_epoch(full_key);
     let prev_epoch = prev_epoch(epoch);
@@ -315,7 +314,6 @@ pub fn next_full_key(full_key: &[u8]) -> Vec<u8> {
 /// compute the prev full key of the given full key
 ///
 /// if the `user_key` has no predecessor key, the result will be a empty vec
-
 pub fn prev_full_key(full_key: &[u8]) -> Vec<u8> {
     let (user_key, epoch) = split_key_epoch(full_key);
     let next_epoch = next_epoch(epoch);
@@ -532,7 +530,7 @@ impl EstimateSize for TableKey<Bytes> {
     }
 }
 
-impl<'a> TableKey<&'a [u8]> {
+impl TableKey<&[u8]> {
     pub fn copy_into<T: CopyFromSlice + AsRef<[u8]>>(&self) -> TableKey<T> {
         TableKey(T::copy_from_slice(self.as_ref()))
     }
@@ -646,7 +644,7 @@ impl<'a> UserKey<&'a [u8]> {
     }
 }
 
-impl<'a, T: AsRef<[u8]> + Clone> UserKey<&'a T> {
+impl<T: AsRef<[u8]> + Clone> UserKey<&T> {
     pub fn cloned(self) -> UserKey<T> {
         UserKey {
             table_id: self.table_id,
@@ -941,7 +939,7 @@ impl EmptySliceRef for Vec<u8> {
 }
 
 const EMPTY_SLICE: &[u8] = b"";
-impl<'a> EmptySliceRef for &'a [u8] {
+impl EmptySliceRef for &[u8] {
     fn empty_slice_ref<'b>() -> &'b Self {
         &EMPTY_SLICE
     }
diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs
index aae0c0c0971ab..366fc02310ebb 100644
--- a/src/storage/hummock_sdk/src/lib.rs
+++ b/src/storage/hummock_sdk/src/lib.rs
@@ -18,7 +18,6 @@
 #![feature(map_many_mut)]
 #![feature(type_alias_impl_trait)]
 #![feature(impl_trait_in_assoc_type)]
-#![feature(is_sorted)]
 #![feature(let_chains)]
 #![feature(btree_cursors)]
 #![feature(strict_overflow_ops)]
diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs
index 324e8a91cf4a3..bbc0ae22148c0 100644
--- a/src/storage/hummock_sdk/src/table_watermark.rs
+++ b/src/storage/hummock_sdk/src/table_watermark.rs
@@ -259,7 +259,7 @@ impl TableWatermarksIndex {
             }
         }
         if self.latest_epoch < committed_epoch {
-            warn!(
+            debug!(
                 latest_epoch = self.latest_epoch,
                 committed_epoch, "committed_epoch exceed table watermark latest_epoch"
             );
diff --git a/src/storage/hummock_sdk/src/version.rs b/src/storage/hummock_sdk/src/version.rs
index 64206d9b45b55..80276b09ffdff 100644
--- a/src/storage/hummock_sdk/src/version.rs
+++ b/src/storage/hummock_sdk/src/version.rs
@@ -21,12 +21,12 @@ use std::sync::{Arc, LazyLock};
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
 use risingwave_common::util::epoch::INVALID_EPOCH;
-use risingwave_pb::hummock::group_delta::PbDeltaType;
+use risingwave_pb::hummock::group_delta::{DeltaType, PbDeltaType};
 use risingwave_pb::hummock::hummock_version_delta::PbGroupDeltas;
 use risingwave_pb::hummock::{
     CompactionConfig, PbGroupConstruct, PbGroupDelta, PbGroupDestroy, PbGroupMerge,
-    PbHummockVersion, PbHummockVersionDelta, PbIntraLevelDelta, PbSstableInfo, PbStateTableInfo,
-    StateTableInfo, StateTableInfoDelta,
+    PbHummockVersion, PbHummockVersionDelta, PbIntraLevelDelta, PbNewL0SubLevel, PbSstableInfo,
+    PbStateTableInfo, StateTableInfo, StateTableInfoDelta,
 };
 use tracing::warn;
 
@@ -512,76 +512,45 @@ impl HummockVersionDelta {
     /// Note: the result can be false positive because we only collect the set of sst object ids in the `inserted_table_infos`,
     /// but it is possible that the object is moved or split from other compaction groups or levels.
     pub fn newly_added_object_ids(&self) -> HashSet<HummockSstableObjectId> {
-        self.group_deltas
-            .values()
-            .flat_map(|group_deltas| {
-                group_deltas.group_deltas.iter().flat_map(|group_delta| {
-                    static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
-                    let sst_slice = if let GroupDelta::IntraLevel(level_delta) = &group_delta {
-                        &level_delta.inserted_table_infos
-                    } else {
-                        &EMPTY_VEC
-                    };
-                    sst_slice.iter().map(|sst| sst.object_id)
-                })
-            })
-            .chain(self.change_log_delta.values().flat_map(|delta| {
-                let new_log = delta.new_log.as_ref().unwrap();
-                new_log
-                    .new_value
-                    .iter()
-                    .map(|sst| sst.object_id)
-                    .chain(new_log.old_value.iter().map(|sst| sst.object_id))
-            }))
+        self.newly_added_sst_infos(None)
+            .map(|sst| sst.object_id)
             .collect()
     }
 
     pub fn newly_added_sst_ids(&self) -> HashSet<HummockSstableObjectId> {
-        let ssts_from_group_deltas = self.group_deltas.values().flat_map(|group_deltas| {
-            group_deltas.group_deltas.iter().flat_map(|group_delta| {
-                static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
-                let sst_slice = if let GroupDelta::IntraLevel(level_delta) = &group_delta {
-                    &level_delta.inserted_table_infos
-                } else {
-                    &EMPTY_VEC
-                };
-                sst_slice.iter()
-            })
-        });
-
-        let ssts_from_change_log = self.change_log_delta.values().flat_map(|delta| {
-            let new_log = delta.new_log.as_ref().unwrap();
-            new_log.new_value.iter().chain(new_log.old_value.iter())
-        });
-
-        ssts_from_group_deltas
-            .chain(ssts_from_change_log)
+        self.newly_added_sst_infos(None)
             .map(|sst| sst.sst_id)
             .collect()
     }
 
     pub fn newly_added_sst_infos<'a>(
         &'a self,
-        select_group: &'a HashSet<CompactionGroupId>,
-    ) -> impl Iterator<Item = &SstableInfo> + 'a {
+        select_group: Option<&'a HashSet<CompactionGroupId>>,
+    ) -> impl Iterator<Item = &'a SstableInfo> + 'a {
         self.group_deltas
             .iter()
-            .filter_map(|(cg_id, group_deltas)| {
-                if select_group.contains(cg_id) {
-                    Some(group_deltas)
-                } else {
+            .filter_map(move |(cg_id, group_deltas)| {
+                if let Some(select_group) = select_group
+                    && !select_group.contains(cg_id)
+                {
                     None
+                } else {
+                    Some(group_deltas)
                 }
             })
             .flat_map(|group_deltas| {
                 group_deltas.group_deltas.iter().flat_map(|group_delta| {
-                    static EMPTY_VEC: Vec<SstableInfo> = Vec::new();
-                    let sst_slice = if let GroupDelta::IntraLevel(level_delta) = &group_delta {
-                        &level_delta.inserted_table_infos
-                    } else {
-                        &EMPTY_VEC
+                    let sst_slice = match &group_delta {
+                        GroupDeltaCommon::NewL0SubLevel(inserted_table_infos)
+                        | GroupDeltaCommon::IntraLevel(IntraLevelDeltaCommon {
+                            inserted_table_infos,
+                            ..
+                        }) => Some(inserted_table_infos.iter()),
+                        GroupDeltaCommon::GroupConstruct(_)
+                        | GroupDeltaCommon::GroupDestroy(_)
+                        | GroupDeltaCommon::GroupMerge(_) => None,
                     };
-                    sst_slice.iter()
+                    sst_slice.into_iter().flatten()
                 })
             })
             .chain(self.change_log_delta.values().flat_map(|delta| {
@@ -785,7 +754,7 @@ where
 pub struct IntraLevelDeltaCommon<T> {
     pub level_idx: u32,
     pub l0_sub_level_id: u64,
-    pub removed_table_ids: Vec<u64>,
+    pub removed_table_ids: HashSet<u64>,
     pub inserted_table_infos: Vec<T>,
     pub vnode_partition_count: u32,
 }
@@ -814,7 +783,7 @@ where
         Self {
             level_idx: pb_intra_level_delta.level_idx,
             l0_sub_level_id: pb_intra_level_delta.l0_sub_level_id,
-            removed_table_ids: pb_intra_level_delta.removed_table_ids,
+            removed_table_ids: HashSet::from_iter(pb_intra_level_delta.removed_table_ids),
             inserted_table_infos: pb_intra_level_delta
                 .inserted_table_infos
                 .into_iter()
@@ -833,7 +802,7 @@ where
         Self {
             level_idx: intra_level_delta.level_idx,
             l0_sub_level_id: intra_level_delta.l0_sub_level_id,
-            removed_table_ids: intra_level_delta.removed_table_ids,
+            removed_table_ids: intra_level_delta.removed_table_ids.into_iter().collect(),
             inserted_table_infos: intra_level_delta
                 .inserted_table_infos
                 .into_iter()
@@ -852,7 +821,11 @@ where
         Self {
             level_idx: intra_level_delta.level_idx,
             l0_sub_level_id: intra_level_delta.l0_sub_level_id,
-            removed_table_ids: intra_level_delta.removed_table_ids.clone(),
+            removed_table_ids: intra_level_delta
+                .removed_table_ids
+                .iter()
+                .cloned()
+                .collect(),
             inserted_table_infos: intra_level_delta
                 .inserted_table_infos
                 .iter()
@@ -871,7 +844,9 @@ where
         Self {
             level_idx: pb_intra_level_delta.level_idx,
             l0_sub_level_id: pb_intra_level_delta.l0_sub_level_id,
-            removed_table_ids: pb_intra_level_delta.removed_table_ids.clone(),
+            removed_table_ids: HashSet::from_iter(
+                pb_intra_level_delta.removed_table_ids.iter().cloned(),
+            ),
             inserted_table_infos: pb_intra_level_delta
                 .inserted_table_infos
                 .iter()
@@ -886,7 +861,7 @@ impl IntraLevelDelta {
     pub fn new(
         level_idx: u32,
         l0_sub_level_id: u64,
-        removed_table_ids: Vec<u64>,
+        removed_table_ids: HashSet<u64>,
         inserted_table_infos: Vec<SstableInfo>,
         vnode_partition_count: u32,
     ) -> Self {
@@ -902,6 +877,7 @@ impl IntraLevelDelta {
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum GroupDeltaCommon<T> {
+    NewL0SubLevel(Vec<T>),
     IntraLevel(IntraLevelDeltaCommon<T>),
     GroupConstruct(PbGroupConstruct),
     GroupDestroy(PbGroupDestroy),
@@ -928,6 +904,13 @@ where
             Some(PbDeltaType::GroupMerge(pb_group_merge)) => {
                 GroupDeltaCommon::GroupMerge(pb_group_merge)
             }
+            Some(DeltaType::NewL0SubLevel(pb_new_sub_level)) => GroupDeltaCommon::NewL0SubLevel(
+                pb_new_sub_level
+                    .inserted_table_infos
+                    .into_iter()
+                    .map(T::from)
+                    .collect(),
+            ),
             None => panic!("delta_type is not set"),
         }
     }
@@ -951,6 +934,14 @@ where
             GroupDeltaCommon::GroupMerge(pb_group_merge) => PbGroupDelta {
                 delta_type: Some(PbDeltaType::GroupMerge(pb_group_merge)),
             },
+            GroupDeltaCommon::NewL0SubLevel(new_sub_level) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::NewL0SubLevel(PbNewL0SubLevel {
+                    inserted_table_infos: new_sub_level
+                        .into_iter()
+                        .map(PbSstableInfo::from)
+                        .collect(),
+                })),
+            },
         }
     }
 }
@@ -973,6 +964,11 @@ where
             GroupDeltaCommon::GroupMerge(pb_group_merge) => PbGroupDelta {
                 delta_type: Some(PbDeltaType::GroupMerge(*pb_group_merge)),
             },
+            GroupDeltaCommon::NewL0SubLevel(new_sub_level) => PbGroupDelta {
+                delta_type: Some(PbDeltaType::NewL0SubLevel(PbNewL0SubLevel {
+                    inserted_table_infos: new_sub_level.iter().map(PbSstableInfo::from).collect(),
+                })),
+            },
         }
     }
 }
@@ -995,6 +991,13 @@ where
             Some(PbDeltaType::GroupMerge(pb_group_merge)) => {
                 GroupDeltaCommon::GroupMerge(*pb_group_merge)
             }
+            Some(DeltaType::NewL0SubLevel(pb_new_sub_level)) => GroupDeltaCommon::NewL0SubLevel(
+                pb_new_sub_level
+                    .inserted_table_infos
+                    .iter()
+                    .map(T::from)
+                    .collect(),
+            ),
             None => panic!("delta_type is not set"),
         }
     }
diff --git a/src/storage/hummock_test/src/bin/replay/main.rs b/src/storage/hummock_test/src/bin/replay/main.rs
index 4dce34153f8a5..98ee3ba448ee7 100644
--- a/src/storage/hummock_test/src/bin/replay/main.rs
+++ b/src/storage/hummock_test/src/bin/replay/main.rs
@@ -40,8 +40,8 @@ use risingwave_hummock_trace::{
 use risingwave_meta::hummock::test_utils::setup_compute_env;
 use risingwave_meta::hummock::MockHummockMetaClient;
 use risingwave_object_store::object::build_remote_object_store;
-use risingwave_storage::filter_key_extractor::{
-    FakeRemoteTableAccessor, RpcFilterKeyExtractorManager,
+use risingwave_storage::compaction_catalog_manager::{
+    CompactionCatalogManager, FakeRemoteTableAccessor,
 };
 use risingwave_storage::hummock::{HummockStorage, SstableStore, SstableStoreConfig};
 use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics, ObjectStoreMetrics};
@@ -166,16 +166,14 @@ async fn create_replay_hummock(r: Record, args: &Args) -> Result<impl GlobalRepl
         )
     };
 
-    let key_filter_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new(
-        FakeRemoteTableAccessor {},
-    )));
-
     let storage = HummockStorage::new(
         storage_opts,
         sstable_store,
         hummock_meta_client.clone(),
         notification_client,
-        key_filter_manager,
+        Arc::new(CompactionCatalogManager::new(Box::new(
+            FakeRemoteTableAccessor {},
+        ))),
         state_store_metrics,
         compactor_metrics,
         None,
diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs
index fa89aad609663..3b7054a51d864 100644
--- a/src/storage/hummock_test/src/compactor_tests.rs
+++ b/src/storage/hummock_test/src/compactor_tests.rs
@@ -14,7 +14,7 @@
 
 #[cfg(test)]
 pub(crate) mod tests {
-    use std::collections::{BTreeMap, BTreeSet, HashSet, VecDeque};
+    use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
     use std::ops::Bound;
     use std::sync::Arc;
 
@@ -53,11 +53,13 @@ pub(crate) mod tests {
     use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient};
     use risingwave_pb::hummock::TableOption;
     use risingwave_rpc_client::HummockMetaClient;
-    use risingwave_storage::filter_key_extractor::{
-        FilterKeyExtractorImpl, FilterKeyExtractorManager, FixedLengthFilterKeyExtractor,
-        FullKeyFilterKeyExtractor,
+    use risingwave_storage::compaction_catalog_manager::{
+        CompactionCatalogAgent, CompactionCatalogAgentRef, FilterKeyExtractorImpl,
+        FixedLengthFilterKeyExtractor, MultiFilterKeyExtractor,
+    };
+    use risingwave_storage::hummock::compactor::compactor_runner::{
+        compact_with_agent, CompactorRunner,
     };
-    use risingwave_storage::hummock::compactor::compactor_runner::{compact, CompactorRunner};
     use risingwave_storage::hummock::compactor::fast_compactor_runner::CompactorRunner as FastCompactorRunner;
     use risingwave_storage::hummock::compactor::{
         CompactionExecutor, CompactorContext, DummyCompactionFilter, TaskProgress,
@@ -82,7 +84,10 @@ pub(crate) mod tests {
 
     use crate::get_notification_client_for_test;
     use crate::local_state_store_test_utils::LocalStateStoreTestExt;
-    use crate::test_utils::{register_tables_with_id_for_test, TestIngestBatch};
+    use crate::test_utils::{
+        register_tables_with_id_for_test, update_filter_key_extractor_for_table_ids,
+        TestIngestBatch,
+    };
 
     pub(crate) async fn get_hummock_storage(
         hummock_meta_client: Arc<dyn HummockMetaClient>,
@@ -111,7 +116,7 @@ pub(crate) mod tests {
         .unwrap();
 
         register_tables_with_id_for_test(
-            hummock.filter_key_extractor_manager(),
+            hummock.compaction_catalog_manager_ref(),
             hummock_manager_ref,
             table_ids,
         )
@@ -197,7 +202,7 @@ pub(crate) mod tests {
         }
     }
 
-    fn get_compactor_context(storage: &HummockStorage) -> CompactorContext {
+    pub fn get_compactor_context(storage: &HummockStorage) -> CompactorContext {
         get_compactor_context_impl(storage.storage_opts().clone(), storage.sstable_store())
     }
 
@@ -240,16 +245,6 @@ pub(crate) mod tests {
         )
         .await;
 
-        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
-        {
-            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-                rpc_filter_key_extractor_manager,
-            ) => rpc_filter_key_extractor_manager,
-            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-        };
-        let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        );
         let compact_ctx = get_compactor_context(&storage);
         let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
             hummock_meta_client.clone(),
@@ -276,8 +271,13 @@ pub(crate) mod tests {
         )
         .await;
 
-        // 2. get compact task
+        let compaction_catalog_agent_ref = storage
+            .compaction_catalog_manager_ref()
+            .acquire(vec![table_id])
+            .await
+            .unwrap();
 
+        // 2. get compact task
         let compaction_group_id =
             get_compaction_group_id_by_table_id(hummock_manager_ref.clone(), table_id).await;
         while let Some(compact_task) = hummock_manager_ref
@@ -287,12 +287,12 @@ pub(crate) mod tests {
         {
             // 3. compact
             let (_tx, rx) = tokio::sync::oneshot::channel();
-            let ((result_task, task_stats, object_timestamps), _) = compact(
+            let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
                 compact_ctx.clone(),
                 compact_task.clone(),
                 rx,
                 Box::new(sstable_object_id_manager.clone()),
-                filter_key_extractor_manager.clone(),
+                compaction_catalog_agent_ref.clone(),
             )
             .await;
 
@@ -404,29 +404,6 @@ pub(crate) mod tests {
         }
     }
 
-    pub fn prepare_compactor_and_filter(
-        storage: &HummockStorage,
-        existing_table_id: u32,
-    ) -> (CompactorContext, FilterKeyExtractorManager) {
-        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
-        {
-            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-                rpc_filter_key_extractor_manager,
-            ) => rpc_filter_key_extractor_manager,
-            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-        };
-        rpc_filter_key_extractor_manager.update(
-            existing_table_id,
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
-        );
-
-        let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        );
-
-        (get_compactor_context(storage), filter_key_extractor_manager)
-    }
-
     #[tokio::test]
     async fn test_compaction_drop_all_key() {
         let (env, hummock_manager_ref, cluster_ctl_ref, worker_id) = setup_compute_env(8080).await;
@@ -435,16 +412,18 @@ pub(crate) mod tests {
             worker_id as _,
         ));
 
+        let notification_client = get_notification_client_for_test(
+            env,
+            hummock_manager_ref.clone(),
+            cluster_ctl_ref,
+            worker_id,
+        )
+        .await;
+
         let existing_table_id: u32 = 1;
         let storage_existing_table_id = get_hummock_storage(
             hummock_meta_client.clone(),
-            get_notification_client_for_test(
-                env,
-                hummock_manager_ref.clone(),
-                cluster_ctl_ref,
-                worker_id,
-            )
-            .await,
+            notification_client,
             &hummock_manager_ref,
             &[existing_table_id],
         )
@@ -483,8 +462,6 @@ pub(crate) mod tests {
             .get_sst_ids_by_group_id(compaction_group_id)
             .collect_vec()
             .is_empty());
-
-        // assert_eq!(0, current_version.num_levels(compaction_group_id));
     }
 
     #[tokio::test]
@@ -519,26 +496,17 @@ pub(crate) mod tests {
         let table_id_2 = storage_2.table_id();
         let table_id_set = HashSet::from_iter([table_id_1, table_id_2]);
 
-        let rpc_filter_key_extractor_manager =
-            match global_storage.filter_key_extractor_manager().clone() {
-                FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-                    rpc_filter_key_extractor_manager,
-                ) => rpc_filter_key_extractor_manager,
-                FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-            };
-
-        rpc_filter_key_extractor_manager.update(
-            table_id_1.table_id(),
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
+        update_filter_key_extractor_for_table_ids(
+            global_storage.compaction_catalog_manager_ref(),
+            &[table_id_1.table_id(), table_id_2.table_id()],
         );
 
-        rpc_filter_key_extractor_manager.update(
-            table_id_2.table_id(),
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
-        );
-        let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        );
+        let compaction_catalog_agent_ref = global_storage
+            .compaction_catalog_manager_ref()
+            .acquire(vec![table_id_1.table_id(), table_id_2.table_id()])
+            .await
+            .unwrap();
+
         let compact_ctx = get_compactor_context_impl(
             global_storage.storage_opts().clone(),
             global_storage.sstable_store(),
@@ -639,12 +607,12 @@ pub(crate) mod tests {
 
         // 4. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let ((result_task, task_stats, object_timestamps), _) = compact(
+        let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
             compact_ctx,
             compact_task.clone(),
             rx,
             Box::new(sstable_object_id_manager.clone()),
-            filter_key_extractor_manager,
+            compaction_catalog_agent_ref.clone(),
         )
         .await;
         hummock_manager_ref
@@ -738,14 +706,6 @@ pub(crate) mod tests {
         )
         .await;
 
-        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
-        {
-            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-                rpc_filter_key_extractor_manager,
-            ) => rpc_filter_key_extractor_manager,
-            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-        };
-
         let compact_ctx = get_compactor_context(&storage);
         let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
             hummock_meta_client.clone(),
@@ -754,13 +714,12 @@ pub(crate) mod tests {
                 .clone()
                 .sstable_id_remote_fetch_number,
         ));
-        rpc_filter_key_extractor_manager.update(
-            existing_table_id,
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
-        );
-        let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        );
+
+        let compaction_catalog_agent_ref = storage
+            .compaction_catalog_manager_ref()
+            .acquire(vec![existing_table_id])
+            .await
+            .unwrap();
         // 1. add sstables
         let val = Bytes::from(b"0"[..].to_vec()); // 1 Byte value
 
@@ -844,12 +803,12 @@ pub(crate) mod tests {
 
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let ((result_task, task_stats, object_timestamps), _) = compact(
+        let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
             compact_ctx,
             compact_task.clone(),
             rx,
             Box::new(sstable_object_id_manager.clone()),
-            filter_key_extractor_manager,
+            compaction_catalog_agent_ref.clone(),
         )
         .await;
 
@@ -950,23 +909,21 @@ pub(crate) mod tests {
         )
         .await;
 
-        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
-        {
-            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-                rpc_filter_key_extractor_manager,
-            ) => rpc_filter_key_extractor_manager,
-            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-        };
-
-        rpc_filter_key_extractor_manager.update(
+        let mut multi_filter_key_extractor = MultiFilterKeyExtractor::default();
+        multi_filter_key_extractor.register(
             existing_table_id,
-            Arc::new(FilterKeyExtractorImpl::FixedLength(
-                FixedLengthFilterKeyExtractor::new(TABLE_PREFIX_LEN + key_prefix.len()),
+            FilterKeyExtractorImpl::FixedLength(FixedLengthFilterKeyExtractor::new(
+                TABLE_PREFIX_LEN + key_prefix.len(),
             )),
         );
-        let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        );
+
+        let table_id_to_vnode =
+            HashMap::from_iter([(existing_table_id, VirtualNode::COUNT_FOR_TEST)]);
+        let compaction_catalog_agent_ref = Arc::new(CompactionCatalogAgent::new(
+            FilterKeyExtractorImpl::Multi(multi_filter_key_extractor),
+            table_id_to_vnode,
+        ));
+
         let compact_ctx = get_compactor_context(&storage);
         let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
             hummock_meta_client.clone(),
@@ -1043,12 +1000,12 @@ pub(crate) mod tests {
 
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let ((result_task, task_stats, object_timestamps), _) = compact(
+        let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
             compact_ctx,
             compact_task.clone(),
             rx,
             Box::new(sstable_object_id_manager.clone()),
-            filter_key_extractor_manager,
+            compaction_catalog_agent_ref.clone(),
         )
         .await;
 
@@ -1156,8 +1113,7 @@ pub(crate) mod tests {
             &[existing_table_id],
         )
         .await;
-        let (compact_ctx, filter_key_extractor_manager) =
-            prepare_compactor_and_filter(&storage, existing_table_id);
+        let compact_ctx = get_compactor_context(&storage);
         let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
             hummock_meta_client.clone(),
             storage
@@ -1221,14 +1177,20 @@ pub(crate) mod tests {
             129
         );
 
+        let compaction_catalog_agent_ref = storage
+            .compaction_catalog_manager_ref()
+            .acquire(vec![existing_table_id])
+            .await
+            .unwrap();
+
         // 3. compact
         let (_tx, rx) = tokio::sync::oneshot::channel();
-        let ((result_task, task_stats, object_timestamps), _) = compact(
+        let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
             compact_ctx,
             compact_task.clone(),
             rx,
             Box::new(sstable_object_id_manager.clone()),
-            filter_key_extractor_manager,
+            compaction_catalog_agent_ref.clone(),
         )
         .await;
 
@@ -1328,9 +1290,8 @@ pub(crate) mod tests {
     async fn run_fast_and_normal_runner(
         compact_ctx: CompactorContext,
         task: CompactTask,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
     ) -> (Vec<SstableInfo>, Vec<SstableInfo>) {
-        let multi_filter_key_extractor =
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor));
         let compaction_filter = DummyCompactionFilter {};
         let slow_compact_runner = CompactorRunner::new(
             0,
@@ -1340,10 +1301,11 @@ pub(crate) mod tests {
                 VecDeque::from_iter([5, 6, 7, 8, 9, 10, 11, 12, 13]),
             )),
         );
+
         let fast_compact_runner = FastCompactorRunner::new(
             compact_ctx.clone(),
             task.clone(),
-            multi_filter_key_extractor.clone(),
+            compaction_catalog_agent_ref.clone(),
             Box::new(SharedComapctorObjectIdManager::for_test(
                 VecDeque::from_iter([22, 23, 24, 25, 26, 27, 28, 29]),
             )),
@@ -1352,7 +1314,7 @@ pub(crate) mod tests {
         let (_, ret1, _) = slow_compact_runner
             .run(
                 compaction_filter,
-                multi_filter_key_extractor,
+                compaction_catalog_agent_ref,
                 Arc::new(TaskProgress::default()),
             )
             .await
@@ -1384,7 +1346,9 @@ pub(crate) mod tests {
         )
         .await;
         hummock_manager_ref.get_new_sst_ids(10).await.unwrap();
-        let (compact_ctx, _) = prepare_compactor_and_filter(&storage, existing_table_id);
+        let compact_ctx = get_compactor_context(&storage);
+        let compaction_catalog_agent_ref =
+            CompactionCatalogAgent::for_test(vec![existing_table_id]);
 
         let sstable_store = compact_ctx.sstable_store.clone();
         let capacity = 256 * 1024;
@@ -1433,7 +1397,9 @@ pub(crate) mod tests {
             gc_delete_keys: true,
             ..Default::default()
         };
-        let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await;
+        let (ret, fast_ret) =
+            run_fast_and_normal_runner(compact_ctx.clone(), task, compaction_catalog_agent_ref)
+                .await;
         check_compaction_result(compact_ctx.sstable_store, ret, fast_ret, capacity).await;
     }
 
@@ -1563,7 +1529,9 @@ pub(crate) mod tests {
         )
         .await;
         hummock_manager_ref.get_new_sst_ids(10).await.unwrap();
-        let (compact_ctx, _) = prepare_compactor_and_filter(&storage, existing_table_id);
+        let compact_ctx = get_compactor_context(&storage);
+        let compaction_catalog_agent_ref =
+            CompactionCatalogAgent::for_test(vec![existing_table_id]);
 
         let sstable_store = compact_ctx.sstable_store.clone();
         let capacity = 256 * 1024;
@@ -1585,7 +1553,6 @@ pub(crate) mod tests {
         );
         let mut sst_infos = vec![];
         let mut max_sst_file_size = 0;
-
         for object_id in 1..3 {
             let mut builder = SstableBuilder::<_, BlockedXor16FilterBuilder>::new(
                 object_id,
@@ -1594,7 +1561,7 @@ pub(crate) mod tests {
                     .create_sst_writer(object_id, SstableWriterOptions::default()),
                 BlockedXor16FilterBuilder::create(opts.bloom_false_positive, opts.capacity / 16),
                 opts.clone(),
-                Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
+                compaction_catalog_agent_ref.clone(),
                 None,
             );
             let mut last_k: u64 = 1;
@@ -1659,7 +1626,12 @@ pub(crate) mod tests {
             gc_delete_keys: true,
             ..Default::default()
         };
-        let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await;
+        let (ret, fast_ret) = run_fast_and_normal_runner(
+            compact_ctx.clone(),
+            task,
+            compaction_catalog_agent_ref.clone(),
+        )
+        .await;
         check_compaction_result(compact_ctx.sstable_store, ret, fast_ret, target_file_size).await;
     }
 
@@ -1685,7 +1657,9 @@ pub(crate) mod tests {
         )
         .await;
         hummock_manager_ref.get_new_sst_ids(10).await.unwrap();
-        let (compact_ctx, _) = prepare_compactor_and_filter(&storage, existing_table_id);
+        let compact_ctx = get_compactor_context(&storage);
+        let compaction_catalog_agent_ref =
+            CompactionCatalogAgent::for_test(vec![existing_table_id]);
 
         let sstable_store = compact_ctx.sstable_store.clone();
         let capacity = 256 * 1024;
@@ -1707,7 +1681,6 @@ pub(crate) mod tests {
         );
         let mut sst_infos = vec![];
         let mut max_sst_file_size = 0;
-
         for object_id in 1..3 {
             let mut builder = SstableBuilder::<_, BlockedXor16FilterBuilder>::new(
                 object_id,
@@ -1716,7 +1689,7 @@ pub(crate) mod tests {
                     .create_sst_writer(object_id, SstableWriterOptions::default()),
                 BlockedXor16FilterBuilder::create(opts.bloom_false_positive, opts.capacity / 16),
                 opts.clone(),
-                Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
+                compaction_catalog_agent_ref.clone(),
                 None,
             );
             let key_count = KEY_COUNT / VirtualNode::COUNT_FOR_TEST * 2;
@@ -1799,7 +1772,7 @@ pub(crate) mod tests {
                     table_infos: sst_infos,
                 },
             ],
-            existing_table_ids: vec![1],
+            existing_table_ids: vec![existing_table_id],
             task_id: 1,
             splits: vec![KeyRange::inf()],
             target_level: 6,
@@ -1810,7 +1783,12 @@ pub(crate) mod tests {
             table_watermarks,
             ..Default::default()
         };
-        let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await;
+        let (ret, fast_ret) = run_fast_and_normal_runner(
+            compact_ctx.clone(),
+            task,
+            compaction_catalog_agent_ref.clone(),
+        )
+        .await;
         let mut fast_tables = Vec::with_capacity(fast_ret.len());
         let mut normal_tables = Vec::with_capacity(ret.len());
         let mut stats = StoreLocalStatistic::default();
@@ -1889,35 +1867,14 @@ pub(crate) mod tests {
         .await;
 
         // basic cg2 -> [1, 2]
-        let rpc_filter_key_extractor_manager = match storage.filter_key_extractor_manager().clone()
-        {
-            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-                rpc_filter_key_extractor_manager,
-            ) => rpc_filter_key_extractor_manager,
-            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-        };
-
         let mut key = BytesMut::default();
         key.put_u16(1);
         key.put_slice(b"key_prefix");
         let key_prefix = key.freeze();
 
-        rpc_filter_key_extractor_manager.update(
-            table_id_1.table_id(),
-            Arc::new(FilterKeyExtractorImpl::FixedLength(
-                FixedLengthFilterKeyExtractor::new(TABLE_PREFIX_LEN + key_prefix.len()),
-            )),
-        );
-        rpc_filter_key_extractor_manager.update(
-            table_id_2.table_id(),
-            Arc::new(FilterKeyExtractorImpl::FixedLength(
-                FixedLengthFilterKeyExtractor::new(TABLE_PREFIX_LEN + key_prefix.len()),
-            )),
-        );
+        let compaction_catalog_agent_ref =
+            CompactionCatalogAgent::for_test(vec![table_id_1.table_id(), table_id_2.table_id()]);
 
-        let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        );
         let compact_ctx = get_compactor_context(&storage);
         let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
             hummock_meta_client.clone(),
@@ -2027,7 +1984,7 @@ pub(crate) mod tests {
             level: usize,
             hummock_manager_ref: HummockManagerRef,
             compact_ctx: CompactorContext,
-            filter_key_extractor_manager: FilterKeyExtractorManager,
+            compaction_catalog_agent_ref: CompactionCatalogAgentRef,
             sstable_object_id_manager: Arc<SstableObjectIdManager>,
         ) {
             // compact left group
@@ -2054,12 +2011,12 @@ pub(crate) mod tests {
 
             // 3. compact
             let (_tx, rx) = tokio::sync::oneshot::channel();
-            let ((result_task, task_stats, object_timestamps), _) = compact(
+            let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
                 compact_ctx,
                 compact_task.clone(),
                 rx,
                 Box::new(sstable_object_id_manager.clone()),
-                filter_key_extractor_manager.clone(),
+                compaction_catalog_agent_ref.clone(),
             )
             .await;
 
@@ -2076,7 +2033,11 @@ pub(crate) mod tests {
         }
 
         let new_cg_id = hummock_manager_ref
-            .move_state_tables_to_dedicated_compaction_group(parent_group_id, &split_table_ids, 0)
+            .move_state_tables_to_dedicated_compaction_group(
+                parent_group_id,
+                &split_table_ids,
+                None,
+            )
             .await
             .unwrap()
             .0;
@@ -2101,7 +2062,7 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
@@ -2111,19 +2072,25 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
 
+        let created_tables = HashSet::from_iter(vec![table_id_1.table_id(), table_id_2.table_id()]);
+
         // try merge
         hummock_manager_ref
-            .merge_compaction_group(parent_group_id, new_cg_id)
+            .merge_compaction_group_for_test(parent_group_id, new_cg_id, created_tables.clone())
             .await
             .unwrap();
 
         let new_cg_id = hummock_manager_ref
-            .move_state_tables_to_dedicated_compaction_group(parent_group_id, &split_table_ids, 0)
+            .move_state_tables_to_dedicated_compaction_group(
+                parent_group_id,
+                &split_table_ids,
+                None,
+            )
             .await
             .unwrap()
             .0;
@@ -2133,7 +2100,7 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
@@ -2143,7 +2110,7 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
@@ -2167,7 +2134,7 @@ pub(crate) mod tests {
 
         // try merge
         hummock_manager_ref
-            .merge_compaction_group(parent_group_id, new_cg_id)
+            .merge_compaction_group_for_test(parent_group_id, new_cg_id, created_tables.clone())
             .await
             .unwrap();
 
@@ -2177,14 +2144,18 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
 
         // try split
         let new_cg_id = hummock_manager_ref
-            .move_state_tables_to_dedicated_compaction_group(parent_group_id, &split_table_ids, 0)
+            .move_state_tables_to_dedicated_compaction_group(
+                parent_group_id,
+                &split_table_ids,
+                None,
+            )
             .await
             .unwrap()
             .0;
@@ -2207,7 +2178,7 @@ pub(crate) mod tests {
         epoch += millisec_interval_epoch;
 
         hummock_manager_ref
-            .merge_compaction_group(parent_group_id, new_cg_id)
+            .merge_compaction_group_for_test(parent_group_id, new_cg_id, created_tables.clone())
             .await
             .unwrap();
 
@@ -2234,7 +2205,7 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
@@ -2244,7 +2215,7 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
@@ -2254,7 +2225,7 @@ pub(crate) mod tests {
             level: usize,
             hummock_manager_ref: HummockManagerRef,
             compact_ctx: CompactorContext,
-            filter_key_extractor_manager: FilterKeyExtractorManager,
+            compaction_catalog_agent_ref: CompactionCatalogAgentRef,
             sstable_object_id_manager: Arc<SstableObjectIdManager>,
         ) {
             loop {
@@ -2279,12 +2250,12 @@ pub(crate) mod tests {
 
                 // 3. compact
                 let (_tx, rx) = tokio::sync::oneshot::channel();
-                let ((result_task, task_stats, object_timestamps), _) = compact(
+                let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
                     compact_ctx.clone(),
                     compact_task.clone(),
                     rx,
                     Box::new(sstable_object_id_manager.clone()),
-                    filter_key_extractor_manager.clone(),
+                    compaction_catalog_agent_ref.clone(),
                 )
                 .await;
 
@@ -2303,7 +2274,11 @@ pub(crate) mod tests {
 
         // try split
         let new_cg_id = hummock_manager_ref
-            .move_state_tables_to_dedicated_compaction_group(parent_group_id, &split_table_ids, 0)
+            .move_state_tables_to_dedicated_compaction_group(
+                parent_group_id,
+                &split_table_ids,
+                None,
+            )
             .await
             .unwrap()
             .0;
@@ -2328,7 +2303,7 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
@@ -2338,14 +2313,14 @@ pub(crate) mod tests {
             0,
             hummock_manager_ref.clone(),
             compact_ctx.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_agent_ref.clone(),
             sstable_object_id_manager.clone(),
         )
         .await;
 
         // try merge
         hummock_manager_ref
-            .merge_compaction_group(parent_group_id, new_cg_id)
+            .merge_compaction_group_for_test(parent_group_id, new_cg_id, created_tables.clone())
             .await
             .unwrap();
     }
diff --git a/src/storage/hummock_test/src/hummock_storage_tests.rs b/src/storage/hummock_test/src/hummock_storage_tests.rs
index 4e6ab26a539c6..24ba1ca1cf779 100644
--- a/src/storage/hummock_test/src/hummock_storage_tests.rs
+++ b/src/storage/hummock_test/src/hummock_storage_tests.rs
@@ -26,7 +26,6 @@ use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_common::range::RangeBoundsExt;
 use risingwave_common::util::epoch::{test_epoch, EpochExt, INVALID_EPOCH};
-use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::key::{
     gen_key_from_bytes, prefixed_range_with_vnode, FullKey, TableKey, UserKey, TABLE_PREFIX_LEN,
 };
@@ -36,6 +35,7 @@ use risingwave_hummock_sdk::table_watermark::{
     TableWatermarksIndex, VnodeWatermark, WatermarkDirection,
 };
 use risingwave_hummock_sdk::{EpochWithGap, LocalSstableInfo};
+use risingwave_meta::hummock::test_utils::get_compaction_group_id_by_table_id;
 use risingwave_meta::hummock::{CommitEpochInfo, NewTableFragmentInfo};
 use risingwave_rpc_client::HummockMetaClient;
 use risingwave_storage::hummock::local_version::pinned_version::PinnedVersion;
@@ -2635,20 +2635,20 @@ async fn test_commit_multi_epoch() {
     commit_epoch(
         epoch1,
         sst1_epoch1.clone(),
-        vec![NewTableFragmentInfo::Normal {
-            mv_table_id: None,
-            internal_table_ids: vec![existing_table_id],
+        vec![NewTableFragmentInfo {
+            table_ids: HashSet::from_iter([existing_table_id]),
         }],
         &[existing_table_id],
     )
     .await;
 
-    let old_cg_id_set: HashSet<_> = {
+    let cg_id =
+        get_compaction_group_id_by_table_id(test_env.manager.clone(), existing_table_id.table_id())
+            .await;
+
+    {
         let version = test_env.manager.get_current_version().await;
-        let cg = version
-            .levels
-            .get(&(StaticCompactionGroupId::StateDefault as _))
-            .unwrap();
+        let cg = version.levels.get(&(cg_id)).unwrap();
         let sub_levels = &cg.l0.sub_levels;
         assert_eq!(sub_levels.len(), 1);
         let sub_level = &sub_levels[0];
@@ -2661,13 +2661,8 @@ async fn test_commit_multi_epoch() {
             .get(&existing_table_id)
             .unwrap();
         assert_eq!(epoch1, info.committed_epoch);
-        assert_eq!(
-            StaticCompactionGroupId::StateDefault as u64,
-            info.compaction_group_id
-        );
-
-        version.levels.keys().cloned().collect()
-    };
+        assert_eq!(cg_id, info.compaction_group_id);
+    }
 
     let sst1_epoch2 = SstableInfo {
         sst_id: 22,
@@ -2684,10 +2679,7 @@ async fn test_commit_multi_epoch() {
 
     {
         let version = test_env.manager.get_current_version().await;
-        let cg = version
-            .levels
-            .get(&(StaticCompactionGroupId::StateDefault as _))
-            .unwrap();
+        let cg = version.levels.get(&(cg_id)).unwrap();
         let sub_levels = &cg.l0.sub_levels;
         assert_eq!(sub_levels.len(), 2);
         let sub_level = &sub_levels[0];
@@ -2703,10 +2695,7 @@ async fn test_commit_multi_epoch() {
             .get(&existing_table_id)
             .unwrap();
         assert_eq!(epoch2, info.committed_epoch);
-        assert_eq!(
-            StaticCompactionGroupId::StateDefault as u64,
-            info.compaction_group_id
-        );
+        assert_eq!(cg_id, info.compaction_group_id);
     };
 
     let new_table_id = TableId::new(2);
@@ -2723,7 +2712,7 @@ async fn test_commit_multi_epoch() {
     commit_epoch(
         epoch1,
         sst2_epoch1.clone(),
-        vec![NewTableFragmentInfo::NewCompactionGroup {
+        vec![NewTableFragmentInfo {
             table_ids: HashSet::from_iter([new_table_id]),
         }],
         &[new_table_id],
@@ -2732,10 +2721,9 @@ async fn test_commit_multi_epoch() {
 
     let new_cg_id = {
         let version = test_env.manager.get_current_version().await;
-        let new_cg_id_set: HashSet<_> = version.levels.keys().cloned().collect();
-        let added_cg_id_set = &new_cg_id_set - &old_cg_id_set;
-        assert_eq!(added_cg_id_set.len(), 1);
-        let new_cg_id = added_cg_id_set.into_iter().next().unwrap();
+        let new_cg_id =
+            get_compaction_group_id_by_table_id(test_env.manager.clone(), new_table_id.table_id())
+                .await;
 
         let new_cg = version.levels.get(&new_cg_id).unwrap();
         let sub_levels = &new_cg.l0.sub_levels;
@@ -2801,10 +2789,7 @@ async fn test_commit_multi_epoch() {
 
     {
         let version = test_env.manager.get_current_version().await;
-        let old_cg = version
-            .levels
-            .get(&(StaticCompactionGroupId::StateDefault as _))
-            .unwrap();
+        let old_cg = version.levels.get(&cg_id).unwrap();
         let sub_levels = &old_cg.l0.sub_levels;
         assert_eq!(sub_levels.len(), 3);
         let sub_level1 = &sub_levels[0];
diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs
index 1d21b74e15738..065c9536006bb 100644
--- a/src/storage/hummock_test/src/sync_point_tests.rs
+++ b/src/storage/hummock_test/src/sync_point_tests.rs
@@ -31,16 +31,16 @@ use risingwave_meta::hummock::compaction::selector::ManualCompactionOption;
 use risingwave_meta::hummock::test_utils::{setup_compute_env, setup_compute_env_with_config};
 use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient};
 use risingwave_rpc_client::HummockMetaClient;
-use risingwave_storage::filter_key_extractor::FilterKeyExtractorManager;
-use risingwave_storage::hummock::compactor::compactor_runner::compact;
+use risingwave_storage::compaction_catalog_manager::CompactionCatalogAgentRef;
+use risingwave_storage::hummock::compactor::compactor_runner::compact_with_agent;
 use risingwave_storage::hummock::compactor::CompactorContext;
 use risingwave_storage::hummock::{CachePolicy, GetObjectId, SstableObjectIdManager};
 use risingwave_storage::store::{LocalStateStore, NewLocalOptions, ReadOptions, StateStoreRead};
 use risingwave_storage::StateStore;
 use serial_test::serial;
 
-use super::compactor_tests::tests::{get_hummock_storage, prepare_compactor_and_filter};
-use crate::compactor_tests::tests::flush_and_commit;
+use super::compactor_tests::tests::get_hummock_storage;
+use crate::compactor_tests::tests::{flush_and_commit, get_compactor_context};
 use crate::get_notification_client_for_test;
 use crate::local_state_store_test_utils::LocalStateStoreTestExt;
 use crate::test_utils::gen_key_from_bytes;
@@ -178,7 +178,7 @@ async fn test_syncpoints_test_failpoints_fetch_ids() {
 pub async fn compact_once(
     hummock_manager_ref: HummockManagerRef,
     compact_ctx: CompactorContext,
-    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compaction_catalog_agent_ref: CompactionCatalogAgentRef,
     sstable_object_id_manager: Arc<SstableObjectIdManager>,
 ) {
     // 2. get compact task
@@ -201,12 +201,12 @@ pub async fn compact_once(
     compact_task.compaction_filter_mask = compaction_filter_flag.bits();
     // 3. compact
     let (_tx, rx) = tokio::sync::oneshot::channel();
-    let ((result_task, task_stats, object_timestamps), _) = compact(
+    let ((result_task, task_stats, object_timestamps), _) = compact_with_agent(
         compact_ctx,
         compact_task.clone(),
         rx,
         Box::new(sstable_object_id_manager),
-        filter_key_extractor_manager.clone(),
+        compaction_catalog_agent_ref.clone(),
     )
     .await;
 
@@ -252,8 +252,13 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
         &[existing_table_id],
     )
     .await;
-    let (compact_ctx, filter_key_extractor_manager) =
-        prepare_compactor_and_filter(&storage, existing_table_id);
+
+    let compact_ctx = get_compactor_context(&storage);
+    let compaction_catalog_agent_ref = storage
+        .compaction_catalog_manager_ref()
+        .acquire(vec![existing_table_id])
+        .await
+        .unwrap();
 
     let sstable_object_id_manager = Arc::new(SstableObjectIdManager::new(
         hummock_meta_client.clone(),
@@ -316,10 +321,11 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
         local.table_id(),
     )
     .await;
+
     compact_once(
         hummock_manager_ref.clone(),
         compact_ctx.clone(),
-        filter_key_extractor_manager.clone(),
+        compaction_catalog_agent_ref.clone(),
         sstable_object_id_manager.clone(),
     )
     .await;
@@ -360,7 +366,7 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
     compact_once(
         hummock_manager_ref.clone(),
         compact_ctx.clone(),
-        filter_key_extractor_manager.clone(),
+        compaction_catalog_agent_ref.clone(),
         sstable_object_id_manager.clone(),
     )
     .await;
@@ -402,7 +408,7 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
     compact_once(
         hummock_manager_ref.clone(),
         compact_ctx.clone(),
-        filter_key_extractor_manager.clone(),
+        compaction_catalog_agent_ref.clone(),
         sstable_object_id_manager.clone(),
     )
     .await;
@@ -437,7 +443,7 @@ async fn test_syncpoints_get_in_delete_range_boundary() {
     compact_once(
         hummock_manager_ref.clone(),
         compact_ctx.clone(),
-        filter_key_extractor_manager.clone(),
+        compaction_catalog_agent_ref.clone(),
         sstable_object_id_manager.clone(),
     )
     .await;
diff --git a/src/storage/hummock_test/src/test_utils.rs b/src/storage/hummock_test/src/test_utils.rs
index 849e77a659f7a..c403917938fb5 100644
--- a/src/storage/hummock_test/src/test_utils.rs
+++ b/src/storage/hummock_test/src/test_utils.rs
@@ -17,6 +17,7 @@ use std::sync::Arc;
 use bytes::Bytes;
 use itertools::Itertools;
 use risingwave_common::catalog::TableId;
+use risingwave_common::hash::VirtualNode;
 use risingwave_common_service::ObserverManager;
 use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId;
 use risingwave_hummock_sdk::key::TableKey;
@@ -29,11 +30,10 @@ use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient};
 use risingwave_meta::manager::MetaSrvEnv;
 use risingwave_pb::catalog::{PbTable, Table};
 use risingwave_rpc_client::HummockMetaClient;
-use risingwave_storage::error::StorageResult;
-use risingwave_storage::filter_key_extractor::{
-    FilterKeyExtractorImpl, FilterKeyExtractorManager, FullKeyFilterKeyExtractor,
-    RpcFilterKeyExtractorManager,
+use risingwave_storage::compaction_catalog_manager::{
+    CompactionCatalogManager, CompactionCatalogManagerRef,
 };
+use risingwave_storage::error::StorageResult;
 use risingwave_storage::hummock::backup_reader::BackupReader;
 use risingwave_storage::hummock::event_handler::HummockVersionUpdate;
 use risingwave_storage::hummock::iterator::test_utils::mock_sstable_store;
@@ -71,7 +71,7 @@ pub async fn prepare_first_valid_version(
     let observer_manager = ObserverManager::new(
         notification_client,
         HummockObserverNode::new(
-            Arc::new(RpcFilterKeyExtractorManager::default()),
+            Arc::new(CompactionCatalogManager::default()),
             backup_manager,
             tx.clone(),
             write_limiter,
@@ -145,7 +145,7 @@ pub async fn with_hummock_storage_v2(
     .unwrap();
 
     register_tables_with_id_for_test(
-        hummock_storage.filter_key_extractor_manager(),
+        hummock_storage.compaction_catalog_manager_ref(),
         &hummock_manager_ref,
         &[table_id.table_id()],
     )
@@ -153,31 +153,28 @@ pub async fn with_hummock_storage_v2(
 
     (hummock_storage, meta_client)
 }
+
 pub fn update_filter_key_extractor_for_table_ids(
-    filter_key_extractor_manager_ref: &FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
     table_ids: &[u32],
 ) {
-    let rpc_filter_key_extractor_manager = match filter_key_extractor_manager_ref {
-        FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        ) => rpc_filter_key_extractor_manager,
-        FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-    };
-
     for table_id in table_ids {
-        rpc_filter_key_extractor_manager.update(
-            *table_id,
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
-        )
+        let mock_table = PbTable {
+            id: *table_id,
+            read_prefix_len_hint: 0,
+            maybe_vnode_count: Some(VirtualNode::COUNT_FOR_TEST as u32),
+            ..Default::default()
+        };
+        compaction_catalog_manager_ref.update(*table_id, mock_table);
     }
 }
 
 pub async fn register_tables_with_id_for_test(
-    filter_key_extractor_manager: &FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
     hummock_manager_ref: &HummockManagerRef,
     table_ids: &[u32],
 ) {
-    update_filter_key_extractor_for_table_ids(filter_key_extractor_manager, table_ids);
+    update_filter_key_extractor_for_table_ids(compaction_catalog_manager_ref, table_ids);
     register_table_ids_to_compaction_group(
         hummock_manager_ref,
         table_ids,
@@ -187,28 +184,19 @@ pub async fn register_tables_with_id_for_test(
 }
 
 pub fn update_filter_key_extractor_for_tables(
-    filter_key_extractor_manager: &FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
     tables: &[PbTable],
 ) {
-    let rpc_filter_key_extractor_manager = match filter_key_extractor_manager {
-        FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            rpc_filter_key_extractor_manager,
-        ) => rpc_filter_key_extractor_manager,
-        FilterKeyExtractorManager::StaticFilterKeyExtractorManager(_) => unreachable!(),
-    };
     for table in tables {
-        rpc_filter_key_extractor_manager.update(
-            table.id,
-            Arc::new(FilterKeyExtractorImpl::from_table(table)),
-        )
+        compaction_catalog_manager_ref.update(table.id, table.clone())
     }
 }
 pub async fn register_tables_with_catalog_for_test(
-    filter_key_extractor_manager: &FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
     hummock_manager_ref: &HummockManagerRef,
     tables: &[Table],
 ) {
-    update_filter_key_extractor_for_tables(filter_key_extractor_manager, tables);
+    update_filter_key_extractor_for_tables(compaction_catalog_manager_ref, tables);
     let table_ids = tables.iter().map(|t| t.id).collect_vec();
     register_table_ids_to_compaction_group(
         hummock_manager_ref,
@@ -233,7 +221,7 @@ impl HummockTestEnv {
 
     pub async fn register_table_id(&self, table_id: TableId) {
         register_tables_with_id_for_test(
-            self.storage.filter_key_extractor_manager(),
+            self.storage.compaction_catalog_manager_ref(),
             &self.manager,
             &[table_id.table_id()],
         )
@@ -243,7 +231,7 @@ impl HummockTestEnv {
 
     pub async fn register_table(&self, table: PbTable) {
         register_tables_with_catalog_for_test(
-            self.storage.filter_key_extractor_manager(),
+            self.storage.compaction_catalog_manager_ref(),
             &self.manager,
             &[table],
         )
diff --git a/src/storage/hummock_trace/src/lib.rs b/src/storage/hummock_trace/src/lib.rs
index 48b0a71010a74..3a2f1dd643f88 100644
--- a/src/storage/hummock_trace/src/lib.rs
+++ b/src/storage/hummock_trace/src/lib.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![feature(cursor_remaining)]
+#![feature(cursor_split)]
 #![feature(trait_alias)]
 #![feature(coroutines)]
 
diff --git a/src/storage/hummock_trace/src/read.rs b/src/storage/hummock_trace/src/read.rs
index bf6563123044c..858e680f35aa3 100644
--- a/src/storage/hummock_trace/src/read.rs
+++ b/src/storage/hummock_trace/src/read.rs
@@ -185,7 +185,8 @@ mod test {
         }
 
         assert!(deserializer.deserialize(&mut buf).is_err());
-        assert!(buf.is_empty());
+        // https://github.com/rust-lang/rust/pull/109174
+        assert!(buf.split().1.is_empty());
     }
 
     #[test]
diff --git a/src/storage/src/filter_key_extractor.rs b/src/storage/src/compaction_catalog_manager.rs
similarity index 73%
rename from src/storage/src/filter_key_extractor.rs
rename to src/storage/src/compaction_catalog_manager.rs
index e9326d37dcd8c..3133cae023300 100644
--- a/src/storage/src/filter_key_extractor.rs
+++ b/src/storage/src/compaction_catalog_manager.rs
@@ -16,13 +16,12 @@ use std::collections::{HashMap, HashSet};
 use std::fmt::Debug;
 use std::sync::Arc;
 
-use itertools::Itertools;
 use parking_lot::RwLock;
 use risingwave_common::catalog::ColumnDesc;
-use risingwave_common::hash::VirtualNode;
+use risingwave_common::hash::{VirtualNode, VnodeCountCompat};
 use risingwave_common::util::row_serde::OrderedRowSerde;
 use risingwave_common::util::sort_util::OrderType;
-use risingwave_hummock_sdk::info_in_release;
+use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::key::{get_table_id, TABLE_PREFIX_LEN};
 use risingwave_pb::catalog::Table;
 use risingwave_rpc_client::error::{Result as RpcResult, RpcError};
@@ -51,7 +50,7 @@ impl FilterKeyExtractorImpl {
         if read_prefix_len == 0 || read_prefix_len > table_catalog.get_pk().len() {
             // for now frontend had not infer the table_id_to_filter_key_extractor, so we
             // use FullKeyFilterKeyExtractor
-            FilterKeyExtractorImpl::Dummy(DummyFilterKeyExtractor)
+            FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)
         } else {
             FilterKeyExtractorImpl::Schema(SchemaFilterKeyExtractor::new(table_catalog))
         }
@@ -186,13 +185,11 @@ impl SchemaFilterKeyExtractor {
 
 #[derive(Default)]
 pub struct MultiFilterKeyExtractor {
-    id_to_filter_key_extractor: HashMap<u32, Arc<FilterKeyExtractorImpl>>,
-    // cached state
-    // last_filter_key_extractor_state: Mutex<Option<(u32, Arc<FilterKeyExtractorImpl>)>>,
+    id_to_filter_key_extractor: HashMap<u32, FilterKeyExtractorImpl>,
 }
 
 impl MultiFilterKeyExtractor {
-    pub fn register(&mut self, table_id: u32, filter_key_extractor: Arc<FilterKeyExtractorImpl>) {
+    pub fn register(&mut self, table_id: u32, filter_key_extractor: FilterKeyExtractorImpl) {
         self.id_to_filter_key_extractor
             .insert(table_id, filter_key_extractor);
     }
@@ -259,48 +256,71 @@ impl StateTableAccessor for FakeRemoteTableAccessor {
         )))
     }
 }
-struct FilterKeyExtractorManagerInner {
-    table_id_to_filter_key_extractor: RwLock<HashMap<u32, Arc<FilterKeyExtractorImpl>>>,
+
+/// `CompactionCatalogManager` is a manager to manage all `Table` which used in compaction
+pub struct CompactionCatalogManager {
+    // `table_id_to_catalog` is a map to store all `Table` which used in compaction
+    table_id_to_catalog: RwLock<HashMap<StateTableId, Table>>,
+    // `table_accessor` is a accessor to fetch `Table` from meta when the table not found
     table_accessor: Box<dyn StateTableAccessor>,
 }
 
-impl FilterKeyExtractorManagerInner {
-    fn update(&self, table_id: u32, filter_key_extractor: Arc<FilterKeyExtractorImpl>) {
-        self.table_id_to_filter_key_extractor
-            .write()
-            .insert(table_id, filter_key_extractor);
+impl Default for CompactionCatalogManager {
+    fn default() -> Self {
+        Self::new(Box::<FakeRemoteTableAccessor>::default())
+    }
+}
+
+impl CompactionCatalogManager {
+    pub fn new(table_accessor: Box<dyn StateTableAccessor>) -> Self {
+        Self {
+            table_id_to_catalog: Default::default(),
+            table_accessor,
+        }
+    }
+}
+
+impl CompactionCatalogManager {
+    /// `update` is used to update `Table` in `table_id_to_catalog` from notification
+    pub fn update(&self, table_id: u32, catalog: Table) {
+        self.table_id_to_catalog.write().insert(table_id, catalog);
     }
 
-    fn sync(&self, filter_key_extractor_map: HashMap<u32, Arc<FilterKeyExtractorImpl>>) {
-        let mut guard = self.table_id_to_filter_key_extractor.write();
+    /// `sync` is used to sync all `Table` in `table_id_to_catalog` from notification whole snapshot
+    pub fn sync(&self, catalog_map: HashMap<u32, Table>) {
+        let mut guard = self.table_id_to_catalog.write();
         guard.clear();
-        guard.extend(filter_key_extractor_map);
+        guard.extend(catalog_map);
     }
 
-    fn remove(&self, table_id: u32) {
-        self.table_id_to_filter_key_extractor
-            .write()
-            .remove(&table_id);
+    /// `remove` is used to remove `Table` in `table_id_to_catalog` by `table_id`
+    pub fn remove(&self, table_id: u32) {
+        self.table_id_to_catalog.write().remove(&table_id);
     }
 
-    async fn acquire(
+    /// `acquire` is used to acquire `CompactionCatalogAgent` by `table_ids`
+    /// if the table not found in `table_id_to_catalog`, it will fetch from meta
+    pub async fn acquire(
         &self,
-        mut table_id_set: HashSet<u32>,
-    ) -> HummockResult<FilterKeyExtractorImpl> {
-        if table_id_set.is_empty() {
+        mut table_ids: Vec<StateTableId>,
+    ) -> HummockResult<CompactionCatalogAgentRef> {
+        if table_ids.is_empty() {
             // table_id_set is empty
             // the table in sst has been deleted
 
             // use full key as default
-            return Ok(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor));
+            return Err(HummockError::other("table_id_set is empty"));
         }
 
         let mut multi_filter_key_extractor = MultiFilterKeyExtractor::default();
+        let mut table_id_to_vnode = HashMap::new();
         {
-            let guard = self.table_id_to_filter_key_extractor.read();
-            table_id_set.retain(|table_id| match guard.get(table_id) {
-                Some(filter_key_extractor) => {
-                    multi_filter_key_extractor.register(*table_id, filter_key_extractor.clone());
+            let guard = self.table_id_to_catalog.read();
+            table_ids.retain(|table_id| match guard.get(table_id) {
+                Some(table_catalog) => {
+                    multi_filter_key_extractor
+                        .register(*table_id, FilterKeyExtractorImpl::from_table(table_catalog));
+                    table_id_to_vnode.insert(*table_id, table_catalog.vnode_count());
                     false
                 }
 
@@ -308,8 +328,7 @@ impl FilterKeyExtractorManagerInner {
             });
         }
 
-        if !table_id_set.is_empty() {
-            let table_ids = table_id_set.iter().cloned().collect_vec();
+        if !table_ids.is_empty() {
             let mut state_tables =
                 self.table_accessor
                     .get_tables(&table_ids)
@@ -320,122 +339,117 @@ impl FilterKeyExtractorManagerInner {
                             e.as_report()
                         ))
                     })?;
-            let mut guard = self.table_id_to_filter_key_extractor.write();
+
+            let mut guard = self.table_id_to_catalog.write();
             for table_id in table_ids {
                 if let Some(table) = state_tables.remove(&table_id) {
-                    let key_extractor = Arc::new(FilterKeyExtractorImpl::from_table(&table));
-                    guard.insert(table_id, key_extractor.clone());
+                    let table_id = table.id;
+                    let key_extractor = FilterKeyExtractorImpl::from_table(&table);
+                    let vnode = table.vnode_count();
+                    guard.insert(table_id, table);
                     multi_filter_key_extractor.register(table_id, key_extractor);
+                    table_id_to_vnode.insert(table_id, vnode);
                 }
             }
         }
 
-        Ok(FilterKeyExtractorImpl::Multi(multi_filter_key_extractor))
+        Ok(Arc::new(CompactionCatalogAgent::new(
+            FilterKeyExtractorImpl::Multi(multi_filter_key_extractor),
+            table_id_to_vnode,
+        )))
     }
-}
 
-/// `RpcFilterKeyExtractorManager` is a wrapper for inner, and provide a protected read and write
-/// interface, its thread safe
-pub struct RpcFilterKeyExtractorManager {
-    inner: FilterKeyExtractorManagerInner,
-}
+    /// `build_compaction_catalog_agent` is used to build `CompactionCatalogAgent` by `table_catalogs`
+    pub fn build_compaction_catalog_agent(
+        table_catalogs: HashMap<StateTableId, Table>,
+    ) -> CompactionCatalogAgentRef {
+        let mut multi_filter_key_extractor = MultiFilterKeyExtractor::default();
+        let mut table_id_to_vnode = HashMap::new();
+        for (table_id, table_catalog) in table_catalogs {
+            multi_filter_key_extractor
+                .register(table_id, FilterKeyExtractorImpl::from_table(&table_catalog));
+            table_id_to_vnode.insert(table_id, table_catalog.vnode_count());
+        }
 
-impl Default for RpcFilterKeyExtractorManager {
-    fn default() -> Self {
-        Self::new(Box::<FakeRemoteTableAccessor>::default())
+        Arc::new(CompactionCatalogAgent::new(
+            FilterKeyExtractorImpl::Multi(multi_filter_key_extractor),
+            table_id_to_vnode,
+        ))
     }
 }
 
-impl RpcFilterKeyExtractorManager {
-    pub fn new(table_accessor: Box<dyn StateTableAccessor>) -> Self {
+/// `CompactionCatalogAgent` is a wrapper of `filter_key_extractor_manager` and `table_id_to_vnode`
+/// The `CompactionCatalogAgent` belongs to a compaction task call, which we will build from the `table_ids` contained in a compact task and use it during the compaction.
+/// The `CompactionCatalogAgent` can act as a agent for the `CompactionCatalogManager`, providing `extract` and `vnode_count` capabilities.
+pub struct CompactionCatalogAgent {
+    filter_key_extractor_manager: FilterKeyExtractorImpl,
+    table_id_to_vnode: HashMap<StateTableId, usize>,
+}
+
+impl CompactionCatalogAgent {
+    pub fn new(
+        filter_key_extractor_manager: FilterKeyExtractorImpl,
+        table_id_to_vnode: HashMap<StateTableId, usize>,
+    ) -> Self {
         Self {
-            inner: FilterKeyExtractorManagerInner {
-                table_id_to_filter_key_extractor: Default::default(),
-                table_accessor,
-            },
+            filter_key_extractor_manager,
+            table_id_to_vnode,
         }
     }
 
-    /// Insert (`table_id`, `filter_key_extractor`) as mapping to `HashMap` for `acquire`
-    pub fn update(&self, table_id: u32, filter_key_extractor: Arc<FilterKeyExtractorImpl>) {
-        info_in_release!("update key extractor of {}", table_id);
-        self.inner.update(table_id, filter_key_extractor);
+    pub fn dummy() -> Self {
+        Self {
+            filter_key_extractor_manager: FilterKeyExtractorImpl::Dummy(DummyFilterKeyExtractor),
+            table_id_to_vnode: Default::default(),
+        }
     }
 
-    /// Remove a mapping by `table_id`
-    pub fn remove(&self, table_id: u32) {
-        info_in_release!("remove key extractor of {}", table_id);
-        self.inner.remove(table_id);
-    }
+    pub fn for_test(table_ids: Vec<StateTableId>) -> Arc<Self> {
+        let full_key_filter_key_extractor =
+            FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor);
 
-    /// Sync all filter key extractors by snapshot
-    pub fn sync(&self, filter_key_extractor_map: HashMap<u32, Arc<FilterKeyExtractorImpl>>) {
-        self.inner.sync(filter_key_extractor_map)
-    }
+        let table_id_to_vnode = table_ids
+            .into_iter()
+            .map(|table_id| (table_id, VirtualNode::COUNT_FOR_TEST))
+            .collect();
 
-    /// Acquire a `MultiFilterKeyExtractor` by `table_id_set`
-    /// Internally, try to get all `filter_key_extractor` from `hashmap`. Will block the caller if
-    /// `table_id` does not util version update (notify), and retry to get
-    async fn acquire(&self, table_id_set: HashSet<u32>) -> HummockResult<FilterKeyExtractorImpl> {
-        self.inner.acquire(table_id_set).await
+        Arc::new(CompactionCatalogAgent::new(
+            full_key_filter_key_extractor,
+            table_id_to_vnode,
+        ))
     }
 }
 
-#[derive(Clone)]
-pub enum FilterKeyExtractorManager {
-    RpcFilterKeyExtractorManager(Arc<RpcFilterKeyExtractorManager>),
-    StaticFilterKeyExtractorManager(Arc<StaticFilterKeyExtractorManager>),
-}
-
-impl FilterKeyExtractorManager {
-    pub async fn acquire(
-        &self,
-        table_id_set: HashSet<u32>,
-    ) -> HummockResult<FilterKeyExtractorImpl> {
-        match self {
-            FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-                rpc_filter_key_exactor_manager,
-            ) => rpc_filter_key_exactor_manager.acquire(table_id_set).await,
-            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(
-                static_filter_key_extractor_manager,
-            ) => static_filter_key_extractor_manager.acquire(table_id_set),
-        }
+impl CompactionCatalogAgent {
+    pub fn extract<'a>(&self, full_key: &'a [u8]) -> &'a [u8] {
+        self.filter_key_extractor_manager.extract(full_key)
     }
-}
 
-#[derive(Clone)]
-pub struct StaticFilterKeyExtractorManager {
-    id_to_table: HashMap<u32, Table>,
-}
+    pub fn vnode_count(&self, table_id: StateTableId) -> usize {
+        *self.table_id_to_vnode.get(&table_id).unwrap_or_else(|| {
+            panic!(
+                "table_id not found {} all_table_ids {:?}",
+                table_id,
+                self.table_id_to_vnode.keys()
+            )
+        })
+    }
 
-impl StaticFilterKeyExtractorManager {
-    pub fn new(id_to_table: HashMap<u32, Table>) -> Self {
-        Self { id_to_table }
+    pub fn table_id_to_vnode_ref(&self) -> &HashMap<StateTableId, usize> {
+        &self.table_id_to_vnode
     }
 
-    fn acquire(&self, table_id_set: HashSet<u32>) -> HummockResult<FilterKeyExtractorImpl> {
-        let mut multi_filter_key_extractor = MultiFilterKeyExtractor::default();
-        for table_id in table_id_set {
-            if let Some(table) = self.id_to_table.get(&table_id) {
-                let key_extractor = Arc::new(FilterKeyExtractorImpl::from_table(table));
-                multi_filter_key_extractor.register(table_id, key_extractor);
-            } else {
-                return Err(HummockError::other(format!(
-                    "table {} is absent in id_to_table, need to request rpc list_tables to get the schema", table_id,
-                )));
-            }
-        }
-        Ok(FilterKeyExtractorImpl::Multi(multi_filter_key_extractor))
+    pub fn table_ids(&self) -> impl Iterator<Item = StateTableId> + '_ {
+        self.table_id_to_vnode.keys().cloned()
     }
 }
 
-pub type FilterKeyExtractorManagerRef = Arc<RpcFilterKeyExtractorManager>;
+pub type CompactionCatalogManagerRef = Arc<CompactionCatalogManager>;
+pub type CompactionCatalogAgentRef = Arc<CompactionCatalogAgent>;
 
 #[cfg(test)]
 mod tests {
-    use std::collections::HashSet;
     use std::mem;
-    use std::sync::Arc;
 
     use bytes::{BufMut, BytesMut};
     use itertools::Itertools;
@@ -453,9 +467,8 @@ mod tests {
     use risingwave_pb::plan_common::PbColumnCatalog;
 
     use super::{DummyFilterKeyExtractor, FilterKeyExtractor, SchemaFilterKeyExtractor};
-    use crate::filter_key_extractor::{
+    use crate::compaction_catalog_manager::{
         FilterKeyExtractorImpl, FullKeyFilterKeyExtractor, MultiFilterKeyExtractor,
-        RpcFilterKeyExtractorManager,
     };
     const fn dummy_vnode() -> [u8; VirtualNode::SIZE] {
         VirtualNode::from_index(233).to_be_bytes()
@@ -595,7 +608,7 @@ mod tests {
             let schema_filter_key_extractor = SchemaFilterKeyExtractor::new(&prost_table);
             multi_filter_key_extractor.register(
                 1,
-                Arc::new(FilterKeyExtractorImpl::Schema(schema_filter_key_extractor)),
+                FilterKeyExtractorImpl::Schema(schema_filter_key_extractor),
             );
             let order_types: Vec<OrderType> = vec![OrderType::ascending(), OrderType::ascending()];
             let schema = vec![DataType::Int64, DataType::Varchar];
@@ -632,7 +645,7 @@ mod tests {
             let schema_filter_key_extractor = SchemaFilterKeyExtractor::new(&prost_table);
             multi_filter_key_extractor.register(
                 2,
-                Arc::new(FilterKeyExtractorImpl::Schema(schema_filter_key_extractor)),
+                FilterKeyExtractorImpl::Schema(schema_filter_key_extractor),
             );
             let order_types: Vec<OrderType> = vec![OrderType::ascending(), OrderType::ascending()];
             let schema = vec![DataType::Int64, DataType::Varchar];
@@ -666,27 +679,26 @@ mod tests {
     }
 
     #[tokio::test]
-    async fn test_filter_key_extractor_manager() {
-        let filter_key_extractor_manager = Arc::new(RpcFilterKeyExtractorManager::default());
-
-        filter_key_extractor_manager.update(
-            1,
-            Arc::new(FilterKeyExtractorImpl::Dummy(DummyFilterKeyExtractor)),
-        );
-
-        let remaining_table_id_set = HashSet::from([1]);
-        let multi_filter_key_extractor = filter_key_extractor_manager
-            .acquire(remaining_table_id_set)
-            .await
-            .unwrap();
+    async fn test_compaction_catalog_manager_exception() {
+        let compaction_catalog_manager = super::CompactionCatalogManager::default();
 
-        match multi_filter_key_extractor {
-            FilterKeyExtractorImpl::Multi(multi_filter_key_extractor) => {
-                assert_eq!(1, multi_filter_key_extractor.size());
+        {
+            let ret = compaction_catalog_manager.acquire(vec![]).await;
+            assert!(ret.is_err());
+            if let Err(e) = ret {
+                assert_eq!(e.to_string(), "Other error: table_id_set is empty");
             }
+        }
 
-            _ => {
-                unreachable!()
+        {
+            // network error with FakeRemoteTableAccessor
+            let ret = compaction_catalog_manager.acquire(vec![1]).await;
+            assert!(ret.is_err());
+            if let Err(e) = ret {
+                assert_eq!(
+                    e.to_string(),
+                    "Other error: request rpc list_tables for meta failed: fake accessor does not support fetch remote table"
+                );
             }
         }
     }
diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs
index 3b032123f426a..5e22d3e45701b 100644
--- a/src/storage/src/hummock/compactor/compaction_utils.rs
+++ b/src/storage/src/hummock/compactor/compaction_utils.rs
@@ -33,7 +33,7 @@ use risingwave_pb::hummock::{BloomFilterType, PbLevelType, PbTableSchema};
 use tokio::time::Instant;
 
 pub use super::context::CompactorContext;
-use crate::filter_key_extractor::FilterKeyExtractorImpl;
+use crate::compaction_catalog_manager::CompactionCatalogAgentRef;
 use crate::hummock::compactor::{
     ConcatSstableIterator, MultiCompactionFilter, StateCleanUpCompactionFilter, TaskProgress,
     TtlCompactionFilter,
@@ -55,7 +55,7 @@ pub struct RemoteBuilderFactory<W: SstableWriterFactory, F: FilterBuilder> {
     pub options: SstableBuilderOptions,
     pub policy: CachePolicy,
     pub remote_rpc_cost: Arc<AtomicU64>,
-    pub filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+    pub compaction_catalog_agent_ref: CompactionCatalogAgentRef,
     pub sstable_writer_factory: W,
     pub _phantom: PhantomData<F>,
 }
@@ -87,7 +87,7 @@ impl<W: SstableWriterFactory, F: FilterBuilder> TableBuilderFactory for RemoteBu
                 self.options.capacity / DEFAULT_ENTRY_SIZE + 1,
             ),
             self.options.clone(),
-            self.filter_key_extractor.clone(),
+            self.compaction_catalog_agent_ref.clone(),
             Some(self.limiter.clone()),
         );
         Ok(builder)
@@ -674,6 +674,6 @@ pub fn calculate_task_parallelism_impl(
     compaction_size: u64,
     max_sub_compaction: u32,
 ) -> usize {
-    let parallelism = (compaction_size + parallel_compact_size - 1) / parallel_compact_size;
+    let parallelism = compaction_size.div_ceil(parallel_compact_size);
     worker_num.min(parallelism.min(max_sub_compaction as u64) as usize)
 }
diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs
index 57b2b929db411..f6b1ee832375f 100644
--- a/src/storage/src/hummock/compactor/compactor_runner.rs
+++ b/src/storage/src/hummock/compactor/compactor_runner.rs
@@ -24,6 +24,7 @@ use risingwave_hummock_sdk::compact::{
     compact_task_to_string, estimate_memory_for_compact_task, statistics_compact_task,
 };
 use risingwave_hummock_sdk::compact_task::CompactTask;
+use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker};
 use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon};
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
@@ -40,7 +41,7 @@ use tokio::sync::oneshot::Receiver;
 use super::iterator::MonitoredCompactorIterator;
 use super::task_progress::TaskProgress;
 use super::{CompactionStatistics, TaskConfig};
-use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorManager};
+use crate::compaction_catalog_manager::{CompactionCatalogAgentRef, CompactionCatalogManagerRef};
 use crate::hummock::compactor::compaction_utils::{
     build_multi_compaction_filter, estimate_task_output_capacity, generate_splits_for_task,
     metrics_report_for_task, optimize_by_copy_block,
@@ -134,7 +135,7 @@ impl CompactorRunner {
     pub async fn run(
         &self,
         compaction_filter: impl CompactionFilter,
-        filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
         task_progress: Arc<TaskProgress>,
     ) -> HummockResult<CompactOutput> {
         let iter = self.build_sst_iter(task_progress.clone())?;
@@ -143,7 +144,7 @@ impl CompactorRunner {
             .compact_key_range(
                 iter,
                 compaction_filter,
-                filter_key_extractor,
+                compaction_catalog_agent_ref,
                 Some(task_progress),
                 Some(self.compact_task.task_id),
                 Some(self.split_index),
@@ -302,12 +303,12 @@ pub fn partition_overlapping_sstable_infos(
 
 /// Handles a compaction task and reports its status to hummock manager.
 /// Always return `Ok` and let hummock manager handle errors.
-pub async fn compact(
+pub async fn compact_with_agent(
     compactor_context: CompactorContext,
     mut compact_task: CompactTask,
     mut shutdown_rx: Receiver<()>,
     object_id_getter: Box<dyn GetObjectId>,
-    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compaction_catalog_agent_ref: CompactionCatalogAgentRef,
 ) -> (
     (
         CompactTask,
@@ -330,35 +331,6 @@ pub async fn compact(
         .start_timer();
 
     let multi_filter = build_multi_compaction_filter(&compact_task);
-
-    let existing_table_ids: HashSet<u32> =
-        HashSet::from_iter(compact_task.existing_table_ids.clone());
-    let compact_table_ids = HashSet::from_iter(
-        compact_task
-            .input_ssts
-            .iter()
-            .flat_map(|level| level.table_infos.iter())
-            .flat_map(|sst| sst.table_ids.clone())
-            .filter(|table_id| existing_table_ids.contains(table_id)),
-    );
-
-    let multi_filter_key_extractor = match build_filter_key_extractor(
-        &compact_task,
-        filter_key_extractor_manager,
-        &compact_table_ids,
-    )
-    .await
-    {
-        Some(multi_filter_key_extractor) => multi_filter_key_extractor,
-        None => {
-            let task_status = TaskStatus::ExecuteFailed;
-            return (
-                compact_done(compact_task, context.clone(), vec![], task_status),
-                None,
-            );
-        }
-    };
-
     let mut task_status = TaskStatus::Success;
     let optimize_by_copy_block = optimize_by_copy_block(&compact_task, &context);
 
@@ -446,7 +418,7 @@ pub async fn compact(
         let runner = fast_compactor_runner::CompactorRunner::new(
             context.clone(),
             compact_task.clone(),
-            multi_filter_key_extractor.clone(),
+            compaction_catalog_agent_ref.clone(),
             object_id_getter.clone(),
             task_progress_guard.progress.clone(),
         );
@@ -490,7 +462,7 @@ pub async fn compact(
     }
     for (split_index, _) in compact_task.splits.iter().enumerate() {
         let filter = multi_filter.clone();
-        let multi_filter_key_extractor = multi_filter_key_extractor.clone();
+        let compaction_catalog_agent_ref = compaction_catalog_agent_ref.clone();
         let compactor_runner = CompactorRunner::new(
             split_index,
             compactor_context.clone(),
@@ -500,7 +472,7 @@ pub async fn compact(
         let task_progress = task_progress_guard.progress.clone();
         let runner = async move {
             compactor_runner
-                .run(filter, multi_filter_key_extractor, task_progress)
+                .run(filter, compaction_catalog_agent_ref, task_progress)
                 .await
         };
         let traced = match context.await_tree_reg.as_ref() {
@@ -587,6 +559,93 @@ pub async fn compact(
     )
 }
 
+/// Handles a compaction task and reports its status to hummock manager.
+/// Always return `Ok` and let hummock manager handle errors.
+pub async fn compact(
+    compactor_context: CompactorContext,
+    compact_task: CompactTask,
+    shutdown_rx: Receiver<()>,
+    object_id_getter: Box<dyn GetObjectId>,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
+) -> (
+    (
+        CompactTask,
+        HashMap<u32, TableStats>,
+        HashMap<HummockSstableObjectId, u64>,
+    ),
+    Option<MemoryTracker>,
+) {
+    let existing_table_ids: HashSet<u32> =
+        HashSet::from_iter(compact_task.existing_table_ids.clone());
+    let compact_table_ids = Vec::from_iter(
+        compact_task
+            .input_ssts
+            .iter()
+            .flat_map(|level| level.table_infos.iter())
+            .flat_map(|sst| sst.table_ids.clone())
+            .filter(|table_id| existing_table_ids.contains(table_id))
+            .sorted()
+            .unique(),
+    );
+
+    let compaction_catalog_agent_ref = match compaction_catalog_manager_ref
+        .acquire(compact_table_ids.clone())
+        .await
+    {
+        Ok(compaction_catalog_agent_ref) => {
+            let acquire_table_ids: HashSet<StateTableId> =
+                compaction_catalog_agent_ref.table_ids().collect();
+            if acquire_table_ids.len() != compact_table_ids.len() {
+                let diff = compact_table_ids
+                    .into_iter()
+                    .collect::<HashSet<_>>()
+                    .symmetric_difference(&acquire_table_ids)
+                    .cloned()
+                    .collect::<Vec<_>>();
+                tracing::warn!(
+                    dif= ?diff,
+                    "Some table ids are not acquired."
+                );
+                return (
+                    compact_done(
+                        compact_task,
+                        compactor_context.clone(),
+                        vec![],
+                        TaskStatus::ExecuteFailed,
+                    ),
+                    None,
+                );
+            }
+
+            compaction_catalog_agent_ref
+        }
+        Err(e) => {
+            tracing::warn!(
+                error = %e.as_report(),
+                "Failed to acquire compaction catalog agent"
+            );
+            return (
+                compact_done(
+                    compact_task,
+                    compactor_context.clone(),
+                    vec![],
+                    TaskStatus::ExecuteFailed,
+                ),
+                None,
+            );
+        }
+    };
+
+    compact_with_agent(
+        compactor_context,
+        compact_task,
+        shutdown_rx,
+        object_id_getter,
+        compaction_catalog_agent_ref,
+    )
+    .await
+}
+
 /// Fills in the compact task and tries to report the task result to meta node.
 pub(crate) fn compact_done(
     mut compact_task: CompactTask,
@@ -810,39 +869,6 @@ where
     Ok(compaction_statistics)
 }
 
-async fn build_filter_key_extractor(
-    compact_task: &CompactTask,
-    filter_key_extractor_manager: FilterKeyExtractorManager,
-    compact_table_ids: &HashSet<u32>,
-) -> Option<Arc<FilterKeyExtractorImpl>> {
-    let multi_filter_key_extractor = match filter_key_extractor_manager
-        .acquire(compact_table_ids.clone())
-        .await
-    {
-        Err(e) => {
-            tracing::error!(error = %e.as_report(), "Failed to fetch filter key extractor tables [{:?}], it may caused by some RPC error", compact_task.existing_table_ids);
-            return None;
-        }
-        Ok(extractor) => extractor,
-    };
-
-    if let FilterKeyExtractorImpl::Multi(multi) = &multi_filter_key_extractor {
-        let found_tables = multi.get_existing_table_ids();
-        let removed_tables = compact_table_ids
-            .iter()
-            .filter(|table_id| !found_tables.contains(table_id))
-            .collect_vec();
-        if !removed_tables.is_empty() {
-            tracing::error!("Failed to fetch filter key extractor tables [{:?}. [{:?}] may be removed by meta-service. ", compact_table_ids, removed_tables);
-            return None;
-        }
-    }
-
-    let multi_filter_key_extractor = Arc::new(multi_filter_key_extractor);
-
-    Some(multi_filter_key_extractor)
-}
-
 #[cfg(test)]
 pub mod tests {
     use risingwave_hummock_sdk::can_concat;
diff --git a/src/storage/src/hummock/compactor/fast_compactor_runner.rs b/src/storage/src/hummock/compactor/fast_compactor_runner.rs
index 641d866e544b5..6cdf38261cdea 100644
--- a/src/storage/src/hummock/compactor/fast_compactor_runner.rs
+++ b/src/storage/src/hummock/compactor/fast_compactor_runner.rs
@@ -30,7 +30,7 @@ use risingwave_hummock_sdk::sstable_info::SstableInfo;
 use risingwave_hummock_sdk::table_stats::TableStats;
 use risingwave_hummock_sdk::{can_concat, compact_task_to_string, EpochWithGap, LocalSstableInfo};
 
-use crate::filter_key_extractor::FilterKeyExtractorImpl;
+use crate::compaction_catalog_manager::CompactionCatalogAgentRef;
 use crate::hummock::block_stream::BlockDataStream;
 use crate::hummock::compactor::task_progress::TaskProgress;
 use crate::hummock::compactor::{
@@ -359,7 +359,7 @@ impl CompactorRunner {
     pub fn new(
         context: CompactorContext,
         task: CompactTask,
-        filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
         object_id_getter: Box<dyn GetObjectId>,
         task_progress: Arc<TaskProgress>,
     ) -> Self {
@@ -391,7 +391,7 @@ impl CompactorRunner {
             options,
             policy: task_config.cache_policy,
             remote_rpc_cost: get_id_time,
-            filter_key_extractor,
+            compaction_catalog_agent_ref: compaction_catalog_agent_ref.clone(),
             sstable_writer_factory: factory,
             _phantom: PhantomData,
         };
@@ -403,6 +403,7 @@ impl CompactorRunner {
             context
                 .storage_opts
                 .compactor_concurrent_uploading_sst_count,
+            compaction_catalog_agent_ref,
         );
         assert_eq!(
             task.input_ssts.len(),
diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs
index 3a4c487b217a8..772df4050b66e 100644
--- a/src/storage/src/hummock/compactor/mod.rs
+++ b/src/storage/src/hummock/compactor/mod.rs
@@ -80,8 +80,8 @@ use super::multi_builder::CapacitySplitTableBuilder;
 use super::{
     GetObjectId, HummockResult, SstableBuilderOptions, SstableObjectIdManager, Xor16FilterBuilder,
 };
-use crate::filter_key_extractor::{
-    FilterKeyExtractorImpl, FilterKeyExtractorManager, StaticFilterKeyExtractorManager,
+use crate::compaction_catalog_manager::{
+    CompactionCatalogAgentRef, CompactionCatalogManager, CompactionCatalogManagerRef,
 };
 use crate::hummock::compactor::compaction_utils::calculate_task_parallelism;
 use crate::hummock::compactor::compactor_runner::{compact_and_build_sst, compact_done};
@@ -130,7 +130,7 @@ impl Compactor {
         &self,
         iter: impl HummockIterator<Direction = Forward>,
         compaction_filter: impl CompactionFilter,
-        filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
         task_progress: Option<Arc<TaskProgress>>,
         task_id: Option<HummockCompactionTaskId>,
         split_index: Option<usize>,
@@ -155,7 +155,7 @@ impl Compactor {
                     factory,
                     iter,
                     compaction_filter,
-                    filter_key_extractor,
+                    compaction_catalog_agent_ref,
                     task_progress.clone(),
                     self.object_id_getter.clone(),
                 )
@@ -166,7 +166,7 @@ impl Compactor {
                     factory,
                     iter,
                     compaction_filter,
-                    filter_key_extractor,
+                    compaction_catalog_agent_ref,
                     task_progress.clone(),
                     self.object_id_getter.clone(),
                 )
@@ -230,7 +230,7 @@ impl Compactor {
         writer_factory: F,
         iter: impl HummockIterator<Direction = Forward>,
         compaction_filter: impl CompactionFilter,
-        filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
         task_progress: Option<Arc<TaskProgress>>,
         object_id_getter: Box<dyn GetObjectId>,
     ) -> HummockResult<(Vec<LocalSstableInfo>, CompactionStatistics)> {
@@ -240,7 +240,7 @@ impl Compactor {
             options: self.options.clone(),
             policy: self.task_config.cache_policy,
             remote_rpc_cost: self.get_id_time.clone(),
-            filter_key_extractor,
+            compaction_catalog_agent_ref: compaction_catalog_agent_ref.clone(),
             sstable_writer_factory: writer_factory,
             _phantom: PhantomData,
         };
@@ -253,6 +253,7 @@ impl Compactor {
             self.context
                 .storage_opts
                 .compactor_concurrent_uploading_sst_count,
+            compaction_catalog_agent_ref,
         );
         let compaction_statistics = compact_and_build_sst(
             &mut sst_builder,
@@ -280,7 +281,7 @@ pub fn start_compactor(
     compactor_context: CompactorContext,
     hummock_meta_client: Arc<dyn HummockMetaClient>,
     sstable_object_id_manager: Arc<SstableObjectIdManager>,
-    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
 ) -> (JoinHandle<()>, Sender<()>) {
     type CompactionShutdownMap = Arc<Mutex<HashMap<u64, Sender<()>>>>;
     let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel();
@@ -470,7 +471,7 @@ pub fn start_compactor(
 
                         let meta_client = hummock_meta_client.clone();
                         let sstable_object_id_manager = sstable_object_id_manager.clone();
-                        let filter_key_extractor_manager = filter_key_extractor_manager.clone();
+                        let compaction_catalog_manager_ref = compaction_catalog_manager_ref.clone();
 
                         match event {
                             ResponseEvent::CompactTask(compact_task) => {
@@ -515,14 +516,16 @@ pub fn start_compactor(
                                     let (tx, rx) = tokio::sync::oneshot::channel();
                                     let task_id = compact_task.task_id;
                                     shutdown.lock().unwrap().insert(task_id, tx);
-                                    let ((compact_task, table_stats, object_timestamps), _memory_tracker) =
-                                        compactor_runner::compact(
-                                            context.clone(),
-                                            compact_task,
-                                            rx,
-                                            Box::new(sstable_object_id_manager.clone()),
-                                            filter_key_extractor_manager.clone(),
-                                        ).await;
+
+                                    let ((compact_task, table_stats, object_timestamps), _memory_tracker)= compactor_runner::compact(
+                                        context.clone(),
+                                        compact_task,
+                                        rx,
+                                        Box::new(sstable_object_id_manager.clone()),
+                                        compaction_catalog_manager_ref.clone(),
+                                    )
+                                    .await;
+
                                     shutdown.lock().unwrap().remove(&task_id);
                                     running_task_parallelism.fetch_sub(parallelism as u32, Ordering::SeqCst);
 
@@ -703,16 +706,10 @@ pub fn start_shared_compactor(
                             output_object_ids,
                             task: dispatch_task,
                         } = request.into_inner();
-                        let id_to_tables = tables.into_iter().fold(HashMap::new(), |mut acc, table| {
+                        let table_id_to_catalog = tables.into_iter().fold(HashMap::new(), |mut acc, table| {
                             acc.insert(table.id, table);
                             acc
                         });
-                        let static_filter_key_extractor_manager: Arc<StaticFilterKeyExtractorManager> =
-                            Arc::new(StaticFilterKeyExtractorManager::new(id_to_tables));
-                        let filter_key_extractor_manager =
-                            FilterKeyExtractorManager::StaticFilterKeyExtractorManager(
-                                static_filter_key_extractor_manager,
-                            );
 
                         let mut output_object_ids_deque: VecDeque<_> = VecDeque::new();
                         output_object_ids_deque.extend(output_object_ids);
@@ -725,12 +722,13 @@ pub fn start_shared_compactor(
                                     let task_id = compact_task.task_id;
                                     shutdown.lock().unwrap().insert(task_id, tx);
 
-                                    let ((compact_task, table_stats, object_timestamps), _memory_tracker)= compactor_runner::compact(
+                                    let compaction_catalog_agent_ref = CompactionCatalogManager::build_compaction_catalog_agent(table_id_to_catalog);
+                                    let ((compact_task, table_stats, object_timestamps), _memory_tracker)= compactor_runner::compact_with_agent(
                                         context.clone(),
                                         compact_task,
                                         rx,
                                         Box::new(shared_compactor_object_id_manager),
-                                        filter_key_extractor_manager.clone(),
+                                        compaction_catalog_agent_ref,
                                     )
                                     .await;
                                     shutdown.lock().unwrap().remove(&task_id);
diff --git a/src/storage/src/hummock/compactor/shared_buffer_compact.rs b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
index 6ca42e41e3d92..b7ae13b2c1067 100644
--- a/src/storage/src/hummock/compactor/shared_buffer_compact.rs
+++ b/src/storage/src/hummock/compactor/shared_buffer_compact.rs
@@ -32,7 +32,7 @@ use risingwave_pb::hummock::compact_task;
 use thiserror_ext::AsReport;
 use tracing::{error, warn};
 
-use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorManager};
+use crate::compaction_catalog_manager::{CompactionCatalogAgentRef, CompactionCatalogManagerRef};
 use crate::hummock::compactor::compaction_filter::DummyCompactionFilter;
 use crate::hummock::compactor::context::{await_tree_key, CompactorContext};
 use crate::hummock::compactor::{check_flush_result, CompactOutput, Compactor};
@@ -57,14 +57,14 @@ pub async fn compact(
     context: CompactorContext,
     sstable_object_id_manager: SstableObjectIdManagerRef,
     payload: Vec<ImmutableMemtable>,
-    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
 ) -> HummockResult<UploadTaskOutput> {
     let new_value_payload = payload.clone();
     let new_value_future = async {
         compact_shared_buffer::<true>(
             context.clone(),
             sstable_object_id_manager.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_manager_ref.clone(),
             new_value_payload,
         )
         .map_ok(move |results| results.into_iter())
@@ -84,7 +84,7 @@ pub async fn compact(
             compact_shared_buffer::<false>(
                 context.clone(),
                 sstable_object_id_manager.clone(),
-                filter_key_extractor_manager.clone(),
+                compaction_catalog_manager_ref.clone(),
                 old_value_payload,
             )
             .await
@@ -109,29 +109,26 @@ pub async fn compact(
 async fn compact_shared_buffer<const IS_NEW_VALUE: bool>(
     context: CompactorContext,
     sstable_object_id_manager: SstableObjectIdManagerRef,
-    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
     mut payload: Vec<ImmutableMemtable>,
 ) -> HummockResult<Vec<LocalSstableInfo>> {
     if !IS_NEW_VALUE {
         assert!(payload.iter().all(|imm| imm.has_old_value()));
     }
     // Local memory compaction looks at all key ranges.
-
-    let mut existing_table_ids: HashSet<u32> = payload
+    let existing_table_ids: HashSet<u32> = payload
         .iter()
         .map(|imm| imm.table_id.table_id)
         .dedup()
         .collect();
     assert!(!existing_table_ids.is_empty());
 
-    let multi_filter_key_extractor = filter_key_extractor_manager
-        .acquire(existing_table_ids.clone())
+    let compaction_catalog_agent_ref = compaction_catalog_manager_ref
+        .acquire(existing_table_ids.iter().copied().collect())
         .await?;
-    if let FilterKeyExtractorImpl::Multi(multi) = &multi_filter_key_extractor {
-        existing_table_ids = multi.get_existing_table_ids();
-    }
-    let multi_filter_key_extractor = Arc::new(multi_filter_key_extractor);
-
+    let existing_table_ids = compaction_catalog_agent_ref
+        .table_ids()
+        .collect::<HashSet<_>>();
     payload.retain(|imm| {
         let ret = existing_table_ids.contains(&imm.table_id.table_id);
         if !ret {
@@ -167,7 +164,7 @@ async fn compact_shared_buffer<const IS_NEW_VALUE: bool>(
             forward_iters.push(imm.clone().into_directed_iter::<Forward, IS_NEW_VALUE>());
         }
         let compaction_executor = context.compaction_executor.clone();
-        let multi_filter_key_extractor = multi_filter_key_extractor.clone();
+        let compaction_catalog_agent_ref = compaction_catalog_agent_ref.clone();
         let handle = compaction_executor.spawn({
             static NEXT_SHARED_BUFFER_COMPACT_ID: LazyLock<AtomicUsize> =
                 LazyLock::new(|| AtomicUsize::new(0));
@@ -187,7 +184,7 @@ async fn compact_shared_buffer<const IS_NEW_VALUE: bool>(
             });
             let future = compactor.run(
                 MergeIterator::new(forward_iters),
-                multi_filter_key_extractor,
+                compaction_catalog_agent_ref,
             );
             if let Some(root) = tree_root {
                 root.instrument(future).left_future()
@@ -550,7 +547,7 @@ impl SharedBufferCompactRunner {
     pub async fn run(
         self,
         iter: impl HummockIterator<Direction = Forward>,
-        filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
     ) -> HummockResult<CompactOutput> {
         let dummy_compaction_filter = DummyCompactionFilter {};
         let (ssts, table_stats_map) = self
@@ -558,7 +555,7 @@ impl SharedBufferCompactRunner {
             .compact_key_range(
                 iter,
                 dummy_compaction_filter,
-                filter_key_extractor,
+                compaction_catalog_agent_ref,
                 None,
                 None,
                 None,
diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
index 7a33ed81b4373..908bb45a43fc4 100644
--- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs
+++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs
@@ -37,7 +37,7 @@ use tracing::{debug, error, info, trace, warn};
 
 use super::refiller::{CacheRefillConfig, CacheRefiller};
 use super::{LocalInstanceGuard, LocalInstanceId, ReadVersionMappingType};
-use crate::filter_key_extractor::FilterKeyExtractorManager;
+use crate::compaction_catalog_manager::CompactionCatalogManagerRef;
 use crate::hummock::compactor::{await_tree_key, compact, CompactorContext};
 use crate::hummock::event_handler::refiller::{CacheRefillerEvent, SpawnRefillTask};
 use crate::hummock::event_handler::uploader::{
@@ -207,14 +207,14 @@ pub struct HummockEventHandler {
 async fn flush_imms(
     payload: Vec<ImmutableMemtable>,
     compactor_context: CompactorContext,
-    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
     sstable_object_id_manager: Arc<SstableObjectIdManager>,
 ) -> HummockResult<UploadTaskOutput> {
     compact(
         compactor_context,
         sstable_object_id_manager,
         payload,
-        filter_key_extractor_manager,
+        compaction_catalog_manager_ref,
     )
     .verbose_instrument_await("shared_buffer_compact")
     .await
@@ -225,7 +225,7 @@ impl HummockEventHandler {
         version_update_rx: UnboundedReceiver<HummockVersionUpdate>,
         pinned_version: PinnedVersion,
         compactor_context: CompactorContext,
-        filter_key_extractor_manager: FilterKeyExtractorManager,
+        compaction_catalog_manager_ref: CompactionCatalogManagerRef,
         sstable_object_id_manager: Arc<SstableObjectIdManager>,
         state_store_metrics: Arc<HummockStateStoreMetrics>,
     ) -> Self {
@@ -251,7 +251,7 @@ impl HummockEventHandler {
                 let upload_task_latency = upload_task_latency.clone();
                 let wait_poll_latency = wait_poll_latency.clone();
                 let upload_compactor_context = upload_compactor_context.clone();
-                let filter_key_extractor_manager = filter_key_extractor_manager.clone();
+                let compaction_catalog_manager_ref = compaction_catalog_manager_ref.clone();
                 let sstable_object_id_manager = sstable_object_id_manager.clone();
                 spawn({
                     let future = async move {
@@ -262,7 +262,7 @@ impl HummockEventHandler {
                                 .flat_map(|imms| imms.into_iter())
                                 .collect(),
                             upload_compactor_context.clone(),
-                            filter_key_extractor_manager.clone(),
+                            compaction_catalog_manager_ref.clone(),
                             sstable_object_id_manager.clone(),
                         )
                         .await?;
diff --git a/src/storage/src/hummock/event_handler/uploader/mod.rs b/src/storage/src/hummock/event_handler/uploader/mod.rs
index 6a20f96008db6..8326b3b876890 100644
--- a/src/storage/src/hummock/event_handler/uploader/mod.rs
+++ b/src/storage/src/hummock/event_handler/uploader/mod.rs
@@ -655,6 +655,7 @@ impl TableUnsyncData {
         self.unsync_epochs.insert(epoch, ());
     }
 
+    #[expect(clippy::type_complexity)]
     fn sync(
         &mut self,
         epoch: HummockEpoch,
diff --git a/src/storage/src/hummock/event_handler/uploader/test_utils.rs b/src/storage/src/hummock/event_handler/uploader/test_utils.rs
index 2f711ede7cefc..6eb41bda52071 100644
--- a/src/storage/src/hummock/event_handler/uploader/test_utils.rs
+++ b/src/storage/src/hummock/event_handler/uploader/test_utils.rs
@@ -271,6 +271,7 @@ impl HummockUploader {
     }
 }
 
+#[expect(clippy::type_complexity)]
 pub(crate) fn prepare_uploader_order_test(
     config: &StorageOpts,
     skip_schedule: bool,
diff --git a/src/storage/src/hummock/iterator/merge_inner.rs b/src/storage/src/hummock/iterator/merge_inner.rs
index 54221ee8b70a9..68fbc7876f1e9 100644
--- a/src/storage/src/hummock/iterator/merge_inner.rs
+++ b/src/storage/src/hummock/iterator/merge_inner.rs
@@ -172,7 +172,7 @@ impl<'a, T: Ord> PeekMutGuard<'a, T> {
     }
 }
 
-impl<'a, T: Ord> Deref for PeekMutGuard<'a, T> {
+impl<T: Ord> Deref for PeekMutGuard<'_, T> {
     type Target = T;
 
     fn deref(&self) -> &Self::Target {
@@ -180,13 +180,13 @@ impl<'a, T: Ord> Deref for PeekMutGuard<'a, T> {
     }
 }
 
-impl<'a, T: Ord> DerefMut for PeekMutGuard<'a, T> {
+impl<T: Ord> DerefMut for PeekMutGuard<'_, T> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         self.peek.as_mut().expect("should not be None")
     }
 }
 
-impl<'a, T: Ord> Drop for PeekMutGuard<'a, T> {
+impl<T: Ord> Drop for PeekMutGuard<'_, T> {
     /// When the guard is dropped, if `pop` or `used` is not called before it is dropped, we will
     /// call `PeekMut::pop` on the `PeekMut` and recycle the node to the unused list.
     fn drop(&mut self) {
diff --git a/src/storage/src/hummock/iterator/mod.rs b/src/storage/src/hummock/iterator/mod.rs
index a205baac0aa96..e0a1d433892d1 100644
--- a/src/storage/src/hummock/iterator/mod.rs
+++ b/src/storage/src/hummock/iterator/mod.rs
@@ -129,7 +129,7 @@ pub trait HummockIterator: Send {
     fn seek<'a>(
         &'a mut self,
         key: FullKey<&'a [u8]>,
-    ) -> impl Future<Output = HummockResult<()>> + Send + '_;
+    ) -> impl Future<Output = HummockResult<()>> + Send;
 
     /// take local statistic info from iterator to report metrics.
     fn collect_local_statistic(&self, _stats: &mut StoreLocalStatistic);
diff --git a/src/storage/src/hummock/observer_manager.rs b/src/storage/src/hummock/observer_manager.rs
index a9171005aeaa9..7ba86a6f08977 100644
--- a/src/storage/src/hummock/observer_manager.rs
+++ b/src/storage/src/hummock/observer_manager.rs
@@ -12,9 +12,6 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::HashMap;
-use std::sync::Arc;
-
 use risingwave_common_service::ObserverState;
 use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta};
 use risingwave_hummock_trace::TraceSpan;
@@ -24,13 +21,13 @@ use risingwave_pb::meta::subscribe_response::{Info, Operation};
 use risingwave_pb::meta::SubscribeResponse;
 use tokio::sync::mpsc::UnboundedSender;
 
-use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorManagerRef};
+use crate::compaction_catalog_manager::CompactionCatalogManagerRef;
 use crate::hummock::backup_reader::BackupReaderRef;
 use crate::hummock::event_handler::HummockVersionUpdate;
 use crate::hummock::write_limiter::WriteLimiterRef;
 
 pub struct HummockObserverNode {
-    filter_key_extractor_manager: FilterKeyExtractorManagerRef,
+    compaction_catalog_manager: CompactionCatalogManagerRef,
     backup_reader: BackupReaderRef,
     write_limiter: WriteLimiterRef,
     version_update_sender: UnboundedSender<HummockVersionUpdate>,
@@ -140,13 +137,13 @@ impl ObserverState for HummockObserverNode {
 
 impl HummockObserverNode {
     pub fn new(
-        filter_key_extractor_manager: FilterKeyExtractorManagerRef,
+        compaction_catalog_manager: CompactionCatalogManagerRef,
         backup_reader: BackupReaderRef,
         version_update_sender: UnboundedSender<HummockVersionUpdate>,
         write_limiter: WriteLimiterRef,
     ) -> Self {
         Self {
-            filter_key_extractor_manager,
+            compaction_catalog_manager,
             backup_reader,
             version_update_sender,
             version: 0,
@@ -155,25 +152,19 @@ impl HummockObserverNode {
     }
 
     fn handle_catalog_snapshot(&mut self, tables: Vec<Table>) {
-        let all_filter_key_extractors: HashMap<u32, Arc<FilterKeyExtractorImpl>> = tables
-            .iter()
-            .map(|t| (t.id, Arc::new(FilterKeyExtractorImpl::from_table(t))))
-            .collect();
-        self.filter_key_extractor_manager
-            .sync(all_filter_key_extractors);
+        self.compaction_catalog_manager
+            .sync(tables.into_iter().map(|t| (t.id, t)).collect());
     }
 
     fn handle_catalog_notification(&mut self, operation: Operation, table_catalog: Table) {
         match operation {
             Operation::Add | Operation::Update => {
-                self.filter_key_extractor_manager.update(
-                    table_catalog.id,
-                    Arc::new(FilterKeyExtractorImpl::from_table(&table_catalog)),
-                );
+                self.compaction_catalog_manager
+                    .update(table_catalog.id, table_catalog);
             }
 
             Operation::Delete => {
-                self.filter_key_extractor_manager.remove(table_catalog.id);
+                self.compaction_catalog_manager.remove(table_catalog.id);
             }
 
             _ => panic!("receive an unsupported notify {:?}", operation),
diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs
index 24e7e14e02e0f..0ad46e5e000a2 100644
--- a/src/storage/src/hummock/sstable/builder.rs
+++ b/src/storage/src/hummock/sstable/builder.rs
@@ -12,11 +12,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeSet;
+use std::collections::{BTreeSet, HashMap};
 use std::sync::Arc;
 use std::time::SystemTime;
 
 use bytes::{Bytes, BytesMut};
+use risingwave_hummock_sdk::compaction_group::StateTableId;
 use risingwave_hummock_sdk::key::{user_key, FullKey, MAX_KEY_LEN};
 use risingwave_hummock_sdk::key_range::KeyRange;
 use risingwave_hummock_sdk::sstable_info::SstableInfo;
@@ -29,7 +30,10 @@ use super::{
     BlockBuilder, BlockBuilderOptions, BlockMeta, SstableMeta, SstableWriter, DEFAULT_BLOCK_SIZE,
     DEFAULT_ENTRY_SIZE, DEFAULT_RESTART_INTERVAL, VERSION,
 };
-use crate::filter_key_extractor::{FilterKeyExtractorImpl, FullKeyFilterKeyExtractor};
+use crate::compaction_catalog_manager::{
+    CompactionCatalogAgent, CompactionCatalogAgentRef, FilterKeyExtractorImpl,
+    FullKeyFilterKeyExtractor,
+};
 use crate::hummock::sstable::{utils, FilterBuilder};
 use crate::hummock::value::HummockValue;
 use crate::hummock::{
@@ -98,7 +102,8 @@ pub struct SstableBuilder<W: SstableWriter, F: FilterBuilder> {
     writer: W,
     /// Current block builder.
     block_builder: BlockBuilder,
-    filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+
+    compaction_catalog_agent_ref: CompactionCatalogAgentRef,
     /// Block metadata vec.
     block_metas: Vec<BlockMeta>,
 
@@ -126,13 +131,23 @@ pub struct SstableBuilder<W: SstableWriter, F: FilterBuilder> {
 }
 
 impl<W: SstableWriter> SstableBuilder<W, Xor16FilterBuilder> {
-    pub fn for_test(sstable_id: u64, writer: W, options: SstableBuilderOptions) -> Self {
+    pub fn for_test(
+        sstable_id: u64,
+        writer: W,
+        options: SstableBuilderOptions,
+        table_id_to_vnode: HashMap<StateTableId, usize>,
+    ) -> Self {
+        let compaction_catalog_agent_ref = Arc::new(CompactionCatalogAgent::new(
+            FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor),
+            table_id_to_vnode,
+        ));
+
         Self::new(
             sstable_id,
             writer,
             Xor16FilterBuilder::new(options.capacity / DEFAULT_ENTRY_SIZE + 1),
             options,
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
+            compaction_catalog_agent_ref,
             None,
         )
     }
@@ -144,7 +159,7 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
         writer: W,
         filter_builder: F,
         options: SstableBuilderOptions,
-        filter_key_extractor: Arc<FilterKeyExtractorImpl>,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
         memory_limiter: Option<Arc<MemoryLimiter>>,
     ) -> Self {
         Self {
@@ -163,7 +178,7 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
             raw_value: BytesMut::new(),
             last_full_key: vec![],
             sstable_id,
-            filter_key_extractor,
+            compaction_catalog_agent_ref,
             table_stats: Default::default(),
             last_table_stats: Default::default(),
             epoch_set: BTreeSet::default(),
@@ -340,7 +355,7 @@ impl<W: SstableWriter, F: FilterBuilder> SstableBuilder<W, F> {
 
         let table_id = full_key.user_key.table_id.table_id();
         let mut extract_key = user_key(&self.raw_key);
-        extract_key = self.filter_key_extractor.extract(extract_key);
+        extract_key = self.compaction_catalog_agent_ref.extract(extract_key);
         // add bloom_filter check
         if !extract_key.is_empty() {
             self.filter_builder.add_key(extract_key, table_id);
@@ -689,7 +704,7 @@ impl SstableBuilderOutputStats {
 
 #[cfg(test)]
 pub(super) mod tests {
-    use std::collections::Bound;
+    use std::collections::{Bound, HashMap};
 
     use risingwave_common::catalog::TableId;
     use risingwave_common::hash::VirtualNode;
@@ -698,7 +713,9 @@ pub(super) mod tests {
 
     use super::*;
     use crate::assert_bytes_eq;
-    use crate::filter_key_extractor::{DummyFilterKeyExtractor, MultiFilterKeyExtractor};
+    use crate::compaction_catalog_manager::{
+        CompactionCatalogAgent, DummyFilterKeyExtractor, MultiFilterKeyExtractor,
+    };
     use crate::hummock::iterator::test_utils::mock_sstable_store;
     use crate::hummock::sstable::xor_filter::BlockedXor16FilterBuilder;
     use crate::hummock::test_utils::{
@@ -718,7 +735,8 @@ pub(super) mod tests {
             ..Default::default()
         };
 
-        let b = SstableBuilder::for_test(0, mock_sst_writer(&opt), opt);
+        let table_id_to_vnode = HashMap::from_iter(vec![(0, VirtualNode::COUNT_FOR_TEST)]);
+        let b = SstableBuilder::for_test(0, mock_sst_writer(&opt), opt, table_id_to_vnode);
 
         b.finish().await.unwrap();
     }
@@ -726,7 +744,9 @@ pub(super) mod tests {
     #[tokio::test]
     async fn test_basic() {
         let opt = default_builder_opt_for_test();
-        let mut b = SstableBuilder::for_test(0, mock_sst_writer(&opt), opt);
+
+        let table_id_to_vnode = HashMap::from_iter(vec![(0, VirtualNode::COUNT_FOR_TEST)]);
+        let mut b = SstableBuilder::for_test(0, mock_sst_writer(&opt), opt, table_id_to_vnode);
 
         for i in 0..TEST_KEYS_COUNT {
             b.add_for_test(
@@ -815,24 +835,30 @@ pub(super) mod tests {
             .clone()
             .create_sst_writer(object_id, writer_opts);
         let mut filter = MultiFilterKeyExtractor::default();
-        filter.register(
-            1,
-            Arc::new(FilterKeyExtractorImpl::Dummy(DummyFilterKeyExtractor)),
-        );
+        filter.register(1, FilterKeyExtractorImpl::Dummy(DummyFilterKeyExtractor));
         filter.register(
             2,
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
-        );
-        filter.register(
-            3,
-            Arc::new(FilterKeyExtractorImpl::Dummy(DummyFilterKeyExtractor)),
+            FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor),
         );
+        filter.register(3, FilterKeyExtractorImpl::Dummy(DummyFilterKeyExtractor));
+
+        let table_id_to_vnode = HashMap::from_iter(vec![
+            (1, VirtualNode::COUNT_FOR_TEST),
+            (2, VirtualNode::COUNT_FOR_TEST),
+            (3, VirtualNode::COUNT_FOR_TEST),
+        ]);
+
+        let compaction_catalog_agent_ref = Arc::new(CompactionCatalogAgent::new(
+            FilterKeyExtractorImpl::Multi(filter),
+            table_id_to_vnode,
+        ));
+
         let mut builder = SstableBuilder::new(
             object_id,
             writer,
             BlockedXor16FilterBuilder::new(1024),
             opts,
-            Arc::new(FilterKeyExtractorImpl::Multi(filter)),
+            compaction_catalog_agent_ref,
             None,
         );
 
diff --git a/src/storage/src/hummock/sstable/multi_builder.rs b/src/storage/src/hummock/sstable/multi_builder.rs
index 45870257bc04e..f03bcf057c81f 100644
--- a/src/storage/src/hummock/sstable/multi_builder.rs
+++ b/src/storage/src/hummock/sstable/multi_builder.rs
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::sync::atomic::AtomicU64;
 use std::sync::atomic::Ordering::SeqCst;
 use std::sync::Arc;
@@ -21,11 +21,13 @@ use bytes::Bytes;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
 use num_integer::Integer;
+use risingwave_common::catalog::TableId;
 use risingwave_common::hash::VirtualNode;
 use risingwave_hummock_sdk::key::{FullKey, UserKey};
 use risingwave_hummock_sdk::LocalSstableInfo;
 use tokio::task::JoinHandle;
 
+use crate::compaction_catalog_manager::CompactionCatalogAgentRef;
 use crate::hummock::compactor::task_progress::TaskProgress;
 use crate::hummock::sstable::filter::FilterBuilder;
 use crate::hummock::sstable_store::SstableStoreRef;
@@ -78,6 +80,8 @@ where
     concurrent_upload_join_handle: FuturesUnordered<UploadJoinHandle>,
 
     concurrent_uploading_sst_count: Option<usize>,
+
+    compaction_catalog_agent_ref: CompactionCatalogAgentRef,
 }
 
 impl<F> CapacitySplitTableBuilder<F>
@@ -92,6 +96,7 @@ where
         task_progress: Option<Arc<TaskProgress>>,
         table_vnode_partition: BTreeMap<u32, u32>,
         concurrent_uploading_sst_count: Option<usize>,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
     ) -> Self {
         // TODO(var-vnode): should use value from caller
         let vnode_count = VirtualNode::COUNT_FOR_COMPAT;
@@ -109,10 +114,14 @@ where
             largest_vnode_in_current_partition: vnode_count - 1,
             concurrent_upload_join_handle: FuturesUnordered::new(),
             concurrent_uploading_sst_count,
+            compaction_catalog_agent_ref,
         }
     }
 
-    pub fn for_test(builder_factory: F) -> Self {
+    pub fn for_test(
+        builder_factory: F,
+        compaction_catalog_agent_ref: CompactionCatalogAgentRef,
+    ) -> Self {
         Self {
             builder_factory,
             sst_outputs: Vec::new(),
@@ -126,6 +135,7 @@ where
             largest_vnode_in_current_partition: VirtualNode::MAX_FOR_TEST.to_index(),
             concurrent_upload_join_handle: FuturesUnordered::new(),
             concurrent_uploading_sst_count: None,
+            compaction_catalog_agent_ref,
         }
     }
 
@@ -222,6 +232,11 @@ where
             let new_vnode_partition_count =
                 self.table_vnode_partition.get(&user_key.table_id.table_id);
 
+            self.vnode_count = self
+                .compaction_catalog_agent_ref
+                .vnode_count(user_key.table_id.table_id);
+            self.largest_vnode_in_current_partition = self.vnode_count - 1;
+
             if new_vnode_partition_count.is_some()
                 || self.table_vnode_partition.contains_key(&self.last_table_id)
             {
@@ -366,7 +381,11 @@ impl TableBuilderFactory for LocalTableBuilderFactory {
             .sstable_store
             .clone()
             .create_sst_writer(id, writer_options);
-        let builder = SstableBuilder::for_test(id, writer, self.options.clone());
+        let table_id_to_vnode = HashMap::from_iter(vec![(
+            TableId::default().table_id(),
+            VirtualNode::COUNT_FOR_TEST,
+        )]);
+        let builder = SstableBuilder::for_test(id, writer, self.options.clone(), table_id_to_vnode);
 
         Ok(builder)
     }
@@ -378,6 +397,9 @@ mod tests {
     use risingwave_common::util::epoch::{test_epoch, EpochExt};
 
     use super::*;
+    use crate::compaction_catalog_manager::{
+        CompactionCatalogAgent, FilterKeyExtractorImpl, FullKeyFilterKeyExtractor,
+    };
     use crate::hummock::iterator::test_utils::mock_sstable_store;
     use crate::hummock::test_utils::{default_builder_opt_for_test, test_key_of, test_user_key_of};
     use crate::hummock::DEFAULT_RESTART_INTERVAL;
@@ -394,7 +416,9 @@ mod tests {
             ..Default::default()
         };
         let builder_factory = LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts);
-        let builder = CapacitySplitTableBuilder::for_test(builder_factory);
+        let compaction_catalog_agent_ref = Arc::new(CompactionCatalogAgent::dummy());
+        let builder =
+            CapacitySplitTableBuilder::for_test(builder_factory, compaction_catalog_agent_ref);
         let results = builder.finish().await.unwrap();
         assert!(results.is_empty());
     }
@@ -410,8 +434,11 @@ mod tests {
             bloom_false_positive: 0.1,
             ..Default::default()
         };
+        let compaction_catalog_agent_ref = CompactionCatalogAgent::for_test(vec![0]);
+
         let builder_factory = LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts);
-        let mut builder = CapacitySplitTableBuilder::for_test(builder_factory);
+        let mut builder =
+            CapacitySplitTableBuilder::for_test(builder_factory, compaction_catalog_agent_ref);
 
         for i in 0..table_capacity {
             builder
@@ -434,11 +461,11 @@ mod tests {
     #[tokio::test]
     async fn test_table_seal() {
         let opts = default_builder_opt_for_test();
-        let mut builder = CapacitySplitTableBuilder::for_test(LocalTableBuilderFactory::new(
-            1001,
-            mock_sstable_store().await,
-            opts,
-        ));
+        let compaction_catalog_agent_ref = CompactionCatalogAgent::for_test(vec![0]);
+        let mut builder = CapacitySplitTableBuilder::for_test(
+            LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts),
+            compaction_catalog_agent_ref,
+        );
         let mut epoch = test_epoch(100);
 
         macro_rules! add {
@@ -478,11 +505,11 @@ mod tests {
     #[tokio::test]
     async fn test_initial_not_allowed_split() {
         let opts = default_builder_opt_for_test();
-        let mut builder = CapacitySplitTableBuilder::for_test(LocalTableBuilderFactory::new(
-            1001,
-            mock_sstable_store().await,
-            opts,
-        ));
+        let compaction_catalog_agent_ref = CompactionCatalogAgent::for_test(vec![0]);
+        let mut builder = CapacitySplitTableBuilder::for_test(
+            LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts),
+            compaction_catalog_agent_ref,
+        );
         builder
             .add_full_key_for_test(test_key_of(0).to_ref(), HummockValue::put(b"v"), false)
             .await
@@ -501,25 +528,28 @@ mod tests {
             ..Default::default()
         };
 
-        let table_partition_vnode =
-            BTreeMap::from([(1_u32, 4_u32), (2_u32, 4_u32), (3_u32, 4_u32)]);
-
-        let mut builder = CapacitySplitTableBuilder::new(
-            LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts),
-            Arc::new(CompactorMetrics::unused()),
-            None,
-            table_partition_vnode,
-            None,
-        );
-
-        let mut table_key = VirtualNode::from_index(0).to_be_bytes().to_vec();
-        table_key.extend_from_slice("a".as_bytes());
+        {
+            let table_partition_vnode =
+                BTreeMap::from([(1_u32, 4_u32), (2_u32, 4_u32), (3_u32, 4_u32)]);
+
+            let compaction_catalog_agent_ref =
+                CompactionCatalogAgent::for_test(vec![0, 1, 2, 3, 4, 5]);
+            let mut builder = CapacitySplitTableBuilder::new(
+                LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts.clone()),
+                Arc::new(CompactorMetrics::unused()),
+                None,
+                table_partition_vnode,
+                None,
+                compaction_catalog_agent_ref,
+            );
+
+            let mut table_key = VirtualNode::from_index(0).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
 
-        let switch_builder =
-            builder.check_switch_builder(&UserKey::for_test(TableId::from(1), &table_key));
-        assert!(switch_builder);
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(1), &table_key));
+            assert!(switch_builder);
 
-        {
             let mut table_key = VirtualNode::from_index(62).to_be_bytes().to_vec();
             table_key.extend_from_slice("a".as_bytes());
             let switch_builder =
@@ -537,19 +567,119 @@ mod tests {
             let switch_builder =
                 builder.check_switch_builder(&UserKey::for_test(TableId::from(1), &table_key));
             assert!(switch_builder);
+
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(2), &table_key));
+            assert!(switch_builder);
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(3), &table_key));
+            assert!(switch_builder);
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(4), &table_key));
+            assert!(switch_builder);
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(5), &table_key));
+            assert!(!switch_builder);
         }
 
-        let switch_builder =
-            builder.check_switch_builder(&UserKey::for_test(TableId::from(2), &table_key));
-        assert!(switch_builder);
-        let switch_builder =
-            builder.check_switch_builder(&UserKey::for_test(TableId::from(3), &table_key));
-        assert!(switch_builder);
-        let switch_builder =
-            builder.check_switch_builder(&UserKey::for_test(TableId::from(4), &table_key));
-        assert!(switch_builder);
-        let switch_builder =
-            builder.check_switch_builder(&UserKey::for_test(TableId::from(5), &table_key));
-        assert!(!switch_builder);
+        {
+            // Test different table vnode count
+            let table_partition_vnode =
+                BTreeMap::from([(1_u32, 4_u32), (2_u32, 4_u32), (3_u32, 4_u32)]);
+
+            let table_id_to_vnode = HashMap::from_iter(vec![(1, 64), (2, 128), (3, 256)]);
+            let compaction_catalog_agent_ref = Arc::new(CompactionCatalogAgent::new(
+                FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor),
+                table_id_to_vnode,
+            ));
+
+            let mut builder = CapacitySplitTableBuilder::new(
+                LocalTableBuilderFactory::new(1001, mock_sstable_store().await, opts),
+                Arc::new(CompactorMetrics::unused()),
+                None,
+                table_partition_vnode,
+                None,
+                compaction_catalog_agent_ref,
+            );
+
+            let mut table_key = VirtualNode::from_index(0).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(1), &table_key));
+            assert!(switch_builder);
+
+            let mut table_key = VirtualNode::from_index(15).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(1), &table_key));
+            assert!(!switch_builder);
+
+            let mut table_key = VirtualNode::from_index(16).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(1), &table_key));
+            assert!(switch_builder);
+
+            let mut table_key = VirtualNode::from_index(0).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(2), &table_key));
+            assert!(switch_builder);
+
+            let mut table_key = VirtualNode::from_index(16).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(2), &table_key));
+            assert!(!switch_builder);
+
+            let mut table_key = VirtualNode::from_index(31).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(2), &table_key));
+            assert!(!switch_builder);
+
+            let mut table_key = VirtualNode::from_index(32).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(2), &table_key));
+            assert!(switch_builder);
+
+            let mut table_key = VirtualNode::from_index(64).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(2), &table_key));
+            assert!(switch_builder);
+
+            let mut table_key = VirtualNode::from_index(0).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(3), &table_key));
+            assert!(switch_builder);
+
+            let mut table_key = VirtualNode::from_index(16).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(3), &table_key));
+            assert!(!switch_builder);
+
+            let mut table_key = VirtualNode::from_index(32).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(3), &table_key));
+            assert!(!switch_builder);
+
+            let mut table_key = VirtualNode::from_index(63).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(3), &table_key));
+            assert!(!switch_builder);
+
+            let mut table_key = VirtualNode::from_index(64).to_be_bytes().to_vec();
+            table_key.extend_from_slice("a".as_bytes());
+            let switch_builder =
+                builder.check_switch_builder(&UserKey::for_test(TableId::from(3), &table_key));
+            assert!(switch_builder);
+        }
     }
 }
diff --git a/src/storage/src/hummock/sstable/utils.rs b/src/storage/src/hummock/sstable/utils.rs
index 4f4693d5e60b8..e9211ed0e32c9 100644
--- a/src/storage/src/hummock/sstable/utils.rs
+++ b/src/storage/src/hummock/sstable/utils.rs
@@ -60,7 +60,6 @@ pub fn xxhash64_checksum(data: &[u8]) -> u64 {
 }
 
 /// Verifies the checksum of the data equals the given checksum with xxhash64.
-
 pub fn xxhash64_verify(data: &[u8], checksum: u64) -> HummockResult<()> {
     let data_checksum = xxhash64_checksum(data);
     if data_checksum != checksum {
diff --git a/src/storage/src/hummock/sstable/xor_filter.rs b/src/storage/src/hummock/sstable/xor_filter.rs
index 1df4333fca459..8120184b17247 100644
--- a/src/storage/src/hummock/sstable/xor_filter.rs
+++ b/src/storage/src/hummock/sstable/xor_filter.rs
@@ -442,13 +442,18 @@ impl Clone for XorFilterReader {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashMap;
+
     use foyer::CacheContext;
     use rand::RngCore;
+    use risingwave_common::hash::VirtualNode;
     use risingwave_common::util::epoch::test_epoch;
     use risingwave_hummock_sdk::EpochWithGap;
 
     use super::*;
-    use crate::filter_key_extractor::{FilterKeyExtractorImpl, FullKeyFilterKeyExtractor};
+    use crate::compaction_catalog_manager::{
+        CompactionCatalogAgent, FilterKeyExtractorImpl, FullKeyFilterKeyExtractor,
+    };
     use crate::hummock::iterator::test_utils::mock_sstable_store;
     use crate::hummock::sstable::{SstableBuilder, SstableBuilderOptions};
     use crate::hummock::test_utils::{test_user_key_of, test_value_of, TEST_KEYS_COUNT};
@@ -475,12 +480,19 @@ mod tests {
         let writer = sstable_store
             .clone()
             .create_sst_writer(object_id, writer_opts);
+
+        let table_id_to_vnode = HashMap::from_iter(vec![(0, VirtualNode::COUNT_FOR_TEST)]);
+        let compaction_catalog_agent_ref = Arc::new(CompactionCatalogAgent::new(
+            FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor),
+            table_id_to_vnode,
+        ));
+
         let mut builder = SstableBuilder::new(
             object_id,
             writer,
             BlockedXor16FilterBuilder::create(0.01, 2048),
             opts,
-            Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
+            compaction_catalog_agent_ref,
             None,
         );
         let mut rng = rand::thread_rng();
diff --git a/src/storage/src/hummock/store/hummock_storage.rs b/src/storage/src/hummock/store/hummock_storage.rs
index 665b064181687..073dd9e1dc1c4 100644
--- a/src/storage/src/hummock/store/hummock_storage.rs
+++ b/src/storage/src/hummock/store/hummock_storage.rs
@@ -38,8 +38,10 @@ use tokio::sync::oneshot;
 
 use super::local_hummock_storage::LocalHummockStorage;
 use super::version::{read_filter_for_version, CommittedVersion, HummockVersionReader};
+use crate::compaction_catalog_manager::{
+    CompactionCatalogManager, CompactionCatalogManagerRef, FakeRemoteTableAccessor,
+};
 use crate::error::StorageResult;
-use crate::filter_key_extractor::{FilterKeyExtractorManager, RpcFilterKeyExtractorManager};
 use crate::hummock::backup_reader::{BackupReader, BackupReaderRef};
 use crate::hummock::compactor::{
     new_compaction_await_tree_reg_ref, CompactionAwaitTreeRegRef, CompactorContext,
@@ -90,7 +92,7 @@ pub struct HummockStorage {
 
     context: CompactorContext,
 
-    filter_key_extractor_manager: FilterKeyExtractorManager,
+    compaction_catalog_manager_ref: CompactionCatalogManagerRef,
 
     sstable_object_id_manager: SstableObjectIdManagerRef,
 
@@ -148,7 +150,7 @@ impl HummockStorage {
         sstable_store: SstableStoreRef,
         hummock_meta_client: Arc<dyn HummockMetaClient>,
         notification_client: impl NotificationClient,
-        filter_key_extractor_manager: Arc<RpcFilterKeyExtractorManager>,
+        compaction_catalog_manager_ref: CompactionCatalogManagerRef,
         state_store_metrics: Arc<HummockStateStoreMetrics>,
         compactor_metrics: Arc<CompactorMetrics>,
         await_tree_config: Option<await_tree::Config>,
@@ -170,7 +172,7 @@ impl HummockStorage {
         let observer_manager = ObserverManager::new(
             notification_client,
             HummockObserverNode::new(
-                filter_key_extractor_manager.clone(),
+                compaction_catalog_manager_ref.clone(),
                 backup_reader.clone(),
                 version_update_tx.clone(),
                 write_limiter.clone(),
@@ -191,9 +193,6 @@ impl HummockStorage {
             hummock_meta_client.clone(),
             options.max_version_pinning_duration_sec,
         ));
-        let filter_key_extractor_manager = FilterKeyExtractorManager::RpcFilterKeyExtractorManager(
-            filter_key_extractor_manager.clone(),
-        );
 
         let await_tree_reg = await_tree_config.map(new_compaction_await_tree_reg_ref);
 
@@ -208,7 +207,7 @@ impl HummockStorage {
             version_update_rx,
             pinned_version,
             compactor_context.clone(),
-            filter_key_extractor_manager.clone(),
+            compaction_catalog_manager_ref.clone(),
             sstable_object_id_manager.clone(),
             state_store_metrics.clone(),
         );
@@ -217,7 +216,7 @@ impl HummockStorage {
 
         let instance = Self {
             context: compactor_context,
-            filter_key_extractor_manager: filter_key_extractor_manager.clone(),
+            compaction_catalog_manager_ref: compaction_catalog_manager_ref.clone(),
             sstable_object_id_manager,
             buffer_tracker: hummock_event_handler.buffer_tracker().clone(),
             version_update_notifier_tx: hummock_event_handler.version_update_notifier_tx(),
@@ -536,8 +535,8 @@ impl HummockStorage {
         &self.sstable_object_id_manager
     }
 
-    pub fn filter_key_extractor_manager(&self) -> &FilterKeyExtractorManager {
-        &self.filter_key_extractor_manager
+    pub fn compaction_catalog_manager_ref(&self) -> CompactionCatalogManagerRef {
+        self.compaction_catalog_manager_ref.clone()
     }
 
     pub fn get_memory_limiter(&self) -> Arc<MemoryLimiter> {
@@ -763,12 +762,16 @@ impl HummockStorage {
         hummock_meta_client: Arc<dyn HummockMetaClient>,
         notification_client: impl NotificationClient,
     ) -> HummockResult<Self> {
+        let compaction_catalog_manager = Arc::new(CompactionCatalogManager::new(Box::new(
+            FakeRemoteTableAccessor {},
+        )));
+
         Self::new(
             options,
             sstable_store,
             hummock_meta_client,
             notification_client,
-            Arc::new(RpcFilterKeyExtractorManager::default()),
+            compaction_catalog_manager,
             Arc::new(HummockStateStoreMetrics::unused()),
             Arc::new(CompactorMetrics::unused()),
             None,
diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs
index d0082f21b31f9..a2fcc5a2c786f 100644
--- a/src/storage/src/hummock/store/local_hummock_storage.rs
+++ b/src/storage/src/hummock/store/local_hummock_storage.rs
@@ -58,7 +58,6 @@ use crate::store::*;
 
 /// `LocalHummockStorage` is a handle for a state table shard to access data from and write data to
 /// the hummock state backend. It is created via `HummockStorage::new_local`.
-
 pub struct LocalHummockStorage {
     mem_table: MemTable,
 
@@ -781,7 +780,7 @@ impl<'a> HummockStorageIteratorInner<'a> {
     }
 }
 
-impl<'a> Drop for HummockStorageIteratorInner<'a> {
+impl Drop for HummockStorageIteratorInner<'_> {
     fn drop(&mut self) {
         self.inner
             .collect_local_statistic(&mut self.stats_guard.local_stats);
@@ -863,7 +862,7 @@ impl<'a> HummockStorageRevIteratorInner<'a> {
     }
 }
 
-impl<'a> Drop for HummockStorageRevIteratorInner<'a> {
+impl Drop for HummockStorageRevIteratorInner<'_> {
     fn drop(&mut self) {
         self.inner
             .collect_local_statistic(&mut self.stats_guard.local_stats);
diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs
index 33bc9fc49f753..d38424820c703 100644
--- a/src/storage/src/hummock/store/version.rs
+++ b/src/storage/src/hummock/store/version.rs
@@ -165,8 +165,8 @@ impl StagingVersion {
         table_id: TableId,
         table_key_range: &'a TableKeyRange,
     ) -> (
-        impl Iterator<Item = &ImmutableMemtable> + 'a,
-        impl Iterator<Item = &SstableInfo> + 'a,
+        impl Iterator<Item = &'a ImmutableMemtable> + 'a,
+        impl Iterator<Item = &'a SstableInfo> + 'a,
     ) {
         let (ref left, ref right) = table_key_range;
         let left = left.as_ref().map(|key| TableKey(key.0.as_ref()));
diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs
index ee3da3088dbd5..45d18027c1ebe 100644
--- a/src/storage/src/hummock/test_utils.rs
+++ b/src/storage/src/hummock/test_utils.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::collections::HashMap;
 use std::sync::Arc;
 
 use bytes::Bytes;
@@ -33,7 +34,9 @@ use super::iterator::test_utils::iterator_test_table_key_of;
 use super::{
     HummockResult, InMemWriter, SstableMeta, SstableWriterOptions, DEFAULT_RESTART_INTERVAL,
 };
-use crate::filter_key_extractor::{FilterKeyExtractorImpl, FullKeyFilterKeyExtractor};
+use crate::compaction_catalog_manager::{
+    CompactionCatalogAgent, FilterKeyExtractorImpl, FullKeyFilterKeyExtractor,
+};
 use crate::hummock::shared_buffer::shared_buffer_batch::{
     SharedBufferBatch, SharedBufferItem, SharedBufferValue,
 };
@@ -153,7 +156,11 @@ pub async fn gen_test_sstable_data(
     opts: SstableBuilderOptions,
     kv_iter: impl Iterator<Item = (FullKey<Vec<u8>>, HummockValue<Vec<u8>>)>,
 ) -> (Bytes, SstableMeta) {
-    let mut b = SstableBuilder::for_test(0, mock_sst_writer(&opts), opts);
+    let table_id_to_vnode = HashMap::from_iter(vec![(
+        TableId::default().table_id(),
+        VirtualNode::COUNT_FOR_TEST,
+    )]);
+    let mut b = SstableBuilder::for_test(0, mock_sst_writer(&opts), opts, table_id_to_vnode);
     for (key, value) in kv_iter {
         b.add_for_test(key.to_ref(), value.as_slice())
             .await
@@ -231,12 +238,22 @@ pub async fn gen_test_sstable_impl<B: AsRef<[u8]> + Clone + Default + Eq, F: Fil
     let writer = sstable_store
         .clone()
         .create_sst_writer(object_id, writer_opts);
+
+    let table_id_to_vnode = HashMap::from_iter(vec![(
+        TableId::default().table_id(),
+        VirtualNode::COUNT_FOR_TEST,
+    )]);
+    let compaction_catalog_agent_ref = Arc::new(CompactionCatalogAgent::new(
+        FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor),
+        table_id_to_vnode,
+    ));
+
     let mut b = SstableBuilder::<_, F>::new(
         object_id,
         writer,
         F::create(opts.bloom_false_positive, opts.capacity / 16),
         opts,
-        Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)),
+        compaction_catalog_agent_ref,
         None,
     );
 
diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs
index 779062767c7ae..c23d17a8de307 100644
--- a/src/storage/src/lib.rs
+++ b/src/storage/src/lib.rs
@@ -27,7 +27,6 @@
 #![feature(type_changing_struct_update)]
 #![test_runner(risingwave_test_runner::test_runner::run_failpont_tests)]
 #![feature(assert_matches)]
-#![feature(is_sorted)]
 #![feature(btree_extract_if)]
 #![feature(exact_size_is_empty)]
 #![cfg_attr(coverage, feature(coverage_attribute))]
@@ -52,7 +51,7 @@ pub mod opts;
 pub mod store_impl;
 pub mod table;
 
-pub mod filter_key_extractor;
+pub mod compaction_catalog_manager;
 pub mod mem_table;
 #[cfg(test)]
 #[cfg(feature = "failpoints")]
diff --git a/src/storage/src/store_impl.rs b/src/storage/src/store_impl.rs
index ae9f630f304cb..2008047378479 100644
--- a/src/storage/src/store_impl.rs
+++ b/src/storage/src/store_impl.rs
@@ -26,8 +26,8 @@ use risingwave_common_service::RpcNotificationClient;
 use risingwave_hummock_sdk::HummockSstableObjectId;
 use risingwave_object_store::object::build_remote_object_store;
 
+use crate::compaction_catalog_manager::{CompactionCatalogManager, RemoteTableAccessor};
 use crate::error::StorageResult;
-use crate::filter_key_extractor::{RemoteTableAccessor, RpcFilterKeyExtractorManager};
 use crate::hummock::hummock_meta_client::MonitoredHummockMetaClient;
 use crate::hummock::{
     Block, BlockCacheEventListener, HummockError, HummockStorage, RecentFilter, Sstable,
@@ -764,15 +764,17 @@ impl StateStoreImpl {
                 }));
                 let notification_client =
                     RpcNotificationClient::new(hummock_meta_client.get_inner().clone());
-                let key_filter_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new(
-                    RemoteTableAccessor::new(hummock_meta_client.get_inner().clone()),
-                )));
+                let compaction_catalog_manager_ref =
+                    Arc::new(CompactionCatalogManager::new(Box::new(
+                        RemoteTableAccessor::new(hummock_meta_client.get_inner().clone()),
+                    )));
+
                 let inner = HummockStorage::new(
                     opts.clone(),
                     sstable_store,
                     hummock_meta_client.clone(),
                     notification_client,
-                    key_filter_manager,
+                    compaction_catalog_manager_ref,
                     state_store_metrics.clone(),
                     compactor_metrics.clone(),
                     await_tree_config,
@@ -844,11 +846,13 @@ pub mod boxed_state_store {
     use crate::store::*;
     use crate::store_impl::AsHummock;
 
+    #[expect(elided_named_lifetimes)] // false positive
     #[async_trait::async_trait]
     pub trait DynamicDispatchedStateStoreIter<T: IterItem>: Send {
         async fn try_next(&mut self) -> StorageResult<Option<T::ItemRef<'_>>>;
     }
 
+    #[expect(elided_named_lifetimes)] // false positive
     #[async_trait::async_trait]
     impl<T: IterItem, I: StateStoreIter<T>> DynamicDispatchedStateStoreIter<T> for I {
         async fn try_next(&mut self) -> StorageResult<Option<T::ItemRef<'_>>> {
@@ -954,12 +958,14 @@ pub mod boxed_state_store {
             read_options: ReadOptions,
         ) -> StorageResult<Option<Bytes>>;
 
+        #[expect(elided_named_lifetimes)] // false positive
         async fn iter(
             &self,
             key_range: TableKeyRange,
             read_options: ReadOptions,
         ) -> StorageResult<BoxLocalStateStoreIterStream<'_>>;
 
+        #[expect(elided_named_lifetimes)] // false positive
         async fn rev_iter(
             &self,
             key_range: TableKeyRange,
@@ -1002,6 +1008,7 @@ pub mod boxed_state_store {
             self.get(key, read_options).await
         }
 
+        #[expect(elided_named_lifetimes)] // false positive
         async fn iter(
             &self,
             key_range: TableKeyRange,
@@ -1010,6 +1017,7 @@ pub mod boxed_state_store {
             Ok(Box::new(self.iter(key_range, read_options).await?))
         }
 
+        #[expect(elided_named_lifetimes)] // false positive
         async fn rev_iter(
             &self,
             key_range: TableKeyRange,
diff --git a/src/storage/src/table/batch_table/storage_table.rs b/src/storage/src/table/batch_table/storage_table.rs
index b5049c7ba1cda..a665a37be68e0 100644
--- a/src/storage/src/table/batch_table/storage_table.rs
+++ b/src/storage/src/table/batch_table/storage_table.rs
@@ -52,7 +52,7 @@ use crate::store::{
     TryWaitEpochOptions,
 };
 use crate::table::merge_sort::merge_sort;
-use crate::table::{ChangeLogRow, KeyedRow, TableDistribution, TableIter};
+use crate::table::{ChangeLogRow, KeyedChangeLogRow, KeyedRow, TableDistribution, TableIter};
 use crate::StateStore;
 
 /// [`StorageTableInner`] is the interface accessing relational data in KV(`StateStore`) with
@@ -755,6 +755,7 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
         &self,
         start_epoch: u64,
         end_epoch: HummockReadEpoch,
+        ordered: bool,
     ) -> StorageResult<impl Stream<Item = StorageResult<ChangeLogRow>> + Send + 'static> {
         let pk_prefix = OwnedRow::default();
         let start_key = self.serialize_pk_bound(&pk_prefix, Unbounded, true);
@@ -797,9 +798,14 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInner<S, SD> {
             0 => unreachable!(),
             1 => iterators.into_iter().next().unwrap(),
             // Concat all iterators if not to preserve order.
-            _ => futures::stream::iter(iterators.into_iter().map(Box::pin).collect_vec())
-                .flatten_unordered(1024),
-        };
+            _ if !ordered => {
+                futures::stream::iter(iterators.into_iter().map(Box::pin).collect_vec())
+                    .flatten_unordered(1024)
+            }
+            // Merge all iterators if to preserve order.
+            _ => merge_sort(iterators.into_iter().map(Box::pin).collect()),
+        }
+        .map(|row| row.map(|key_row| key_row.into_owned_row()));
 
         Ok(iter)
     }
@@ -1007,16 +1013,21 @@ impl<S: StateStore, SD: ValueRowSerde> StorageTableInnerIterLogInner<S, SD> {
     }
 
     /// Yield a row with its primary key.
-    fn into_stream(self) -> impl Stream<Item = StorageResult<ChangeLogRow>> {
-        self.iter.into_stream(move |(_key, value)| {
-            value.try_map(|value| {
-                let full_row = self.row_deserializer.deserialize(value)?;
-                let row = self
-                    .mapping
-                    .project(OwnedRow::new(full_row))
-                    .into_owned_row();
-                Ok(row)
-            })
+    fn into_stream(self) -> impl Stream<Item = StorageResult<KeyedChangeLogRow<Bytes>>> {
+        self.iter.into_stream(move |(table_key, value)| {
+            value
+                .try_map(|value| {
+                    let full_row = self.row_deserializer.deserialize(value)?;
+                    let row = self
+                        .mapping
+                        .project(OwnedRow::new(full_row))
+                        .into_owned_row();
+                    Ok(row)
+                })
+                .map(|row| KeyedChangeLogRow {
+                    vnode_prefixed_key: table_key.copy_into(),
+                    row,
+                })
         })
     }
 }
diff --git a/src/storage/src/table/merge_sort.rs b/src/storage/src/table/merge_sort.rs
index d17144364df42..57d5e780fbd65 100644
--- a/src/storage/src/table/merge_sort.rs
+++ b/src/storage/src/table/merge_sort.rs
@@ -19,59 +19,94 @@ use std::error::Error;
 use futures::{Stream, StreamExt};
 use futures_async_stream::try_stream;
 
-use super::KeyedRow;
+use super::{KeyedChangeLogRow, KeyedRow};
 
-struct Node<K: AsRef<[u8]>, S> {
+pub trait NodePeek<R> {
+    fn vnode_key(&self) -> &[u8];
+    fn into_row(self: Box<Self>) -> R;
+}
+
+impl<K: AsRef<[u8]>> NodePeek<KeyedRow<K>> for KeyedRow<K> {
+    fn vnode_key(&self) -> &[u8] {
+        self.key()
+    }
+
+    fn into_row(self: Box<Self>) -> KeyedRow<K> {
+        *self
+    }
+}
+
+impl<K: AsRef<[u8]>> NodePeek<KeyedChangeLogRow<K>> for KeyedChangeLogRow<K> {
+    fn vnode_key(&self) -> &[u8] {
+        self.key()
+    }
+
+    fn into_row(self: Box<Self>) -> KeyedChangeLogRow<K> {
+        *self
+    }
+}
+
+struct Node<'a, S, R> {
     stream: S,
 
     /// The next item polled from `stream` previously. Since the `eq` and `cmp` must be synchronous
     /// functions, we need to implement peeking manually.
-    peeked: KeyedRow<K>,
+    peeked: Box<dyn NodePeek<R> + 'a + Send + Sync>,
 }
 
-impl<K: AsRef<[u8]>, S> PartialEq for Node<K, S> {
+impl<S, R> PartialEq for Node<'_, S, R> {
     fn eq(&self, other: &Self) -> bool {
-        match self.peeked.key() == other.peeked.key() {
+        match self.peeked.vnode_key() == other.peeked.vnode_key() {
             true => unreachable!("primary key from different iters should be unique"),
             false => false,
         }
     }
 }
-impl<K: AsRef<[u8]>, S> Eq for Node<K, S> {}
+impl<S, R> Eq for Node<'_, S, R> {}
 
-impl<K: AsRef<[u8]>, S> PartialOrd for Node<K, S> {
+impl<S, R> PartialOrd for Node<'_, S, R> {
     fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
         Some(self.cmp(other))
     }
 }
 
-impl<K: AsRef<[u8]>, S> Ord for Node<K, S> {
+impl<S, R> Ord for Node<'_, S, R> {
     fn cmp(&self, other: &Self) -> std::cmp::Ordering {
         // The heap is a max heap, so we need to reverse the order.
-        self.peeked.key().cmp(other.peeked.key()).reverse()
+        self.peeked
+            .vnode_key()
+            .cmp(other.peeked.vnode_key())
+            .reverse()
     }
 }
 
-#[try_stream(ok=KeyedRow<K>, error=E)]
-pub async fn merge_sort<'a, K, E, R>(streams: Vec<R>)
+#[try_stream(ok=KO, error=E)]
+pub async fn merge_sort<'a, KO, E, NP, R>(streams: Vec<R>)
 where
-    K: AsRef<[u8]> + 'a,
+    // K: AsRef<[u8]> + 'a,
+    KO: 'a,
+    NP: NodePeek<KO> + 'a + Send + Sync,
     E: Error + 'a,
-    R: Stream<Item = Result<KeyedRow<K>, E>> + 'a + Unpin,
+    R: Stream<Item = Result<NP, E>> + 'a + Unpin,
 {
     let mut heap = BinaryHeap::new();
     for mut stream in streams {
         if let Some(peeked) = stream.next().await.transpose()? {
-            heap.push(Node { stream, peeked });
+            heap.push(Node {
+                stream,
+                peeked: Box::new(peeked),
+            });
         }
     }
     while let Some(mut node) = heap.peek_mut() {
         // Note: If the `next` returns `Err`, we'll fail to yield the previous item.
         yield match node.stream.next().await.transpose()? {
             // There still remains data in the stream, take and update the peeked value.
-            Some(new_peeked) => std::mem::replace(&mut node.peeked, new_peeked),
+            Some(new_peeked) => {
+                std::mem::replace(&mut node.peeked, Box::new(new_peeked)).into_row()
+            }
             // This stream is exhausted, remove it from the heap.
-            None => PeekMut::pop(node).peeked,
+            None => PeekMut::pop(node).peeked.into_row(),
         };
     }
 }
diff --git a/src/storage/src/table/mod.rs b/src/storage/src/table/mod.rs
index ceabcd14ea2d6..c27b4ba96dab6 100644
--- a/src/storage/src/table/mod.rs
+++ b/src/storage/src/table/mod.rs
@@ -150,6 +150,29 @@ impl<T: AsRef<[u8]>> Deref for KeyedRow<T> {
     }
 }
 
+#[derive(Debug)]
+pub struct KeyedChangeLogRow<T: AsRef<[u8]>> {
+    vnode_prefixed_key: TableKey<T>,
+    row: ChangeLogRow,
+}
+
+impl<T: AsRef<[u8]>> KeyedChangeLogRow<T> {
+    pub fn new(table_key: TableKey<T>, row: ChangeLogRow) -> Self {
+        Self {
+            vnode_prefixed_key: table_key,
+            row,
+        }
+    }
+
+    pub fn into_owned_row(self) -> ChangeLogRow {
+        self.row
+    }
+
+    pub fn key(&self) -> &[u8] {
+        self.vnode_prefixed_key.key_part()
+    }
+}
+
 pub type ChangeLogRow = ChangeLogValue<OwnedRow>;
 
 pub fn deserialize_log_stream<'a>(
diff --git a/src/stream/src/cache/managed_lru.rs b/src/stream/src/cache/managed_lru.rs
index 7e1b2aa391306..d498c20e2794a 100644
--- a/src/stream/src/cache/managed_lru.rs
+++ b/src/stream/src/cache/managed_lru.rs
@@ -196,7 +196,7 @@ impl<'a, V: EstimateSize> MutGuard<'a, V> {
     }
 }
 
-impl<'a, V: EstimateSize> Drop for MutGuard<'a, V> {
+impl<V: EstimateSize> Drop for MutGuard<'_, V> {
     fn drop(&mut self) {
         let new_value_size = self.inner.estimated_size();
         if new_value_size != self.old_value_size {
@@ -209,7 +209,7 @@ impl<'a, V: EstimateSize> Drop for MutGuard<'a, V> {
     }
 }
 
-impl<'a, V: EstimateSize> Deref for MutGuard<'a, V> {
+impl<V: EstimateSize> Deref for MutGuard<'_, V> {
     type Target = V;
 
     fn deref(&self) -> &Self::Target {
@@ -217,7 +217,7 @@ impl<'a, V: EstimateSize> Deref for MutGuard<'a, V> {
     }
 }
 
-impl<'a, V: EstimateSize> DerefMut for MutGuard<'a, V> {
+impl<V: EstimateSize> DerefMut for MutGuard<'_, V> {
     fn deref_mut(&mut self) -> &mut Self::Target {
         self.inner
     }
diff --git a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
index 6e020536cfb99..4bfe82aa7c41d 100644
--- a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
+++ b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs
@@ -1191,6 +1191,7 @@ mod tests {
         )
     }
 
+    #[expect(clippy::type_complexity)]
     fn gen_single_test_stream(
         serde: LogStoreRowSerde,
         seq_id: &mut SeqIdType,
diff --git a/src/stream/src/common/state_cache/ordered.rs b/src/stream/src/common/state_cache/ordered.rs
index 8a47d1b7ecafe..02797eabad764 100644
--- a/src/stream/src/common/state_cache/ordered.rs
+++ b/src/stream/src/common/state_cache/ordered.rs
@@ -41,7 +41,10 @@ impl<K: Ord + EstimateSize, V: EstimateSize> Default for OrderedStateCache<K, V>
 }
 
 impl<K: Ord + EstimateSize, V: EstimateSize> StateCache for OrderedStateCache<K, V> {
-    type Filler<'a> = &'a mut Self where Self: 'a;
+    type Filler<'a>
+        = &'a mut Self
+    where
+        Self: 'a;
     type Key = K;
     type Value = V;
 
diff --git a/src/stream/src/common/state_cache/top_n.rs b/src/stream/src/common/state_cache/top_n.rs
index fef7017d75129..6ca988bf2740e 100644
--- a/src/stream/src/common/state_cache/top_n.rs
+++ b/src/stream/src/common/state_cache/top_n.rs
@@ -108,7 +108,10 @@ impl<K: Ord + EstimateSize, V: EstimateSize> TopNStateCache<K, V> {
 }
 
 impl<K: Ord + EstimateSize, V: EstimateSize> StateCache for TopNStateCache<K, V> {
-    type Filler<'a> = &'a mut Self where Self: 'a;
+    type Filler<'a>
+        = &'a mut Self
+    where
+        Self: 'a;
     type Key = K;
     type Value = V;
 
diff --git a/src/stream/src/common/table/state_table.rs b/src/stream/src/common/table/state_table.rs
index 3ada0afd71625..0ecb0eac90718 100644
--- a/src/stream/src/common/table/state_table.rs
+++ b/src/stream/src/common/table/state_table.rs
@@ -29,7 +29,7 @@ use risingwave_common::bitmap::Bitmap;
 use risingwave_common::catalog::{
     get_dist_key_in_pk_indices, ColumnDesc, ColumnId, TableId, TableOption,
 };
-use risingwave_common::hash::{VirtualNode, VnodeBitmapExt};
+use risingwave_common::hash::{IsSingleton, VirtualNode, VnodeBitmapExt};
 use risingwave_common::row::{self, once, CompactedRow, Once, OwnedRow, Row, RowExt};
 use risingwave_common::types::{DataType, Datum, DefaultOrd, DefaultOrdered, ScalarImpl};
 use risingwave_common::util::column_index_mapping::ColIndexMapping;
@@ -363,6 +363,7 @@ where
 
         let distribution =
             TableDistribution::new(vnodes, dist_key_in_pk_indices, vnode_col_idx_in_pk);
+        assert_eq!(distribution.is_singleton(), table_catalog.is_singleton());
 
         let pk_data_types = pk_indices
             .iter()
@@ -643,16 +644,16 @@ where
     pub async fn get_encoded_row(&self, pk: impl Row) -> StreamExecutorResult<Option<Bytes>> {
         assert!(pk.len() <= self.pk_indices.len());
 
-        if self.prefix_hint_len != 0 {
-            debug_assert_eq!(self.prefix_hint_len, pk.len());
-        }
-
         let serialized_pk =
             serialize_pk_with_vnode(&pk, &self.pk_serde, self.compute_vnode_by_pk(&pk));
 
         let prefix_hint = if self.prefix_hint_len != 0 && self.prefix_hint_len == pk.len() {
             Some(serialized_pk.slice(VirtualNode::SIZE..))
         } else {
+            #[cfg(debug_assertions)]
+            if self.prefix_hint_len != 0 {
+                warn!("prefix_hint_len is not equal to pk.len(), may not be able to utilize bloom filter");
+            }
             None
         };
 
diff --git a/src/stream/src/executor/actor.rs b/src/stream/src/executor/actor.rs
index 0aa5ebca9007f..65f7e55cdd4e6 100644
--- a/src/stream/src/executor/actor.rs
+++ b/src/stream/src/executor/actor.rs
@@ -103,9 +103,9 @@ impl ActorContext {
             fragment_id: stream_actor.fragment_id,
             mview_definition: stream_actor.mview_definition.clone(),
             vnode_count: (stream_actor.vnode_bitmap.as_ref())
-                // An unset `vnode_bitmap` means the actor is a singleton.
-                // For backwards compatibility, `VirtualNode::COUNT_FOR_COMPAT` is used for singleton.
-                .map_or(VirtualNode::COUNT_FOR_COMPAT, |b| Bitmap::from(b).len()),
+                // An unset `vnode_bitmap` means the actor is a singleton,
+                // where only `SINGLETON_VNODE` is set.
+                .map_or(1, |b| Bitmap::from(b).len()),
             cur_mem_val: Arc::new(0.into()),
             last_mem_val: Arc::new(0.into()),
             total_mem_val,
diff --git a/src/stream/src/executor/aggregation/agg_group.rs b/src/stream/src/executor/aggregation/agg_group.rs
index 116df36b40906..ba59b60ce68d6 100644
--- a/src/stream/src/executor/aggregation/agg_group.rs
+++ b/src/stream/src/executor/aggregation/agg_group.rs
@@ -207,7 +207,7 @@ impl<S: StateStore, Strtg: Strategy> AggGroup<S, Strtg> {
         row_count_index: usize,
         extreme_cache_size: usize,
         input_schema: &Schema,
-    ) -> StreamExecutorResult<(Self, AggStateCacheStats)> {
+    ) -> StreamExecutorResult<Self> {
         let encoded_states = intermediate_state_table
             .get_row(group_key.as_ref().map(GroupKey::table_pk))
             .await?;
@@ -239,12 +239,11 @@ impl<S: StateStore, Strtg: Strategy> AggGroup<S, Strtg> {
         };
 
         if encoded_states.is_some() {
-            let (_, outputs, stats) = this.get_outputs(storages, agg_funcs).await?;
+            let (_, outputs, _stats) = this.get_outputs(storages, agg_funcs).await?;
             this.prev_outputs = Some(outputs);
-            Ok((this, stats))
-        } else {
-            Ok((this, AggStateCacheStats::default()))
         }
+
+        Ok(this)
     }
 
     /// Create a group from encoded states for EOWC. The previous output is set to `None`.
diff --git a/src/stream/src/executor/aggregation/agg_state_cache.rs b/src/stream/src/executor/aggregation/agg_state_cache.rs
index 8a202ed6dd1dd..dc04b1313ed0b 100644
--- a/src/stream/src/executor/aggregation/agg_state_cache.rs
+++ b/src/stream/src/executor/aggregation/agg_state_cache.rs
@@ -161,7 +161,7 @@ where
     cache_filler: C::Filler<'filler>,
 }
 
-impl<'filler, C> AggStateCacheFiller for GenericAggStateCacheFiller<'filler, C>
+impl<C> AggStateCacheFiller for GenericAggStateCacheFiller<'_, C>
 where
     C: StateCache<Key = CacheKey, Value = CacheValue>,
 {
diff --git a/src/stream/src/executor/backfill/snapshot_backfill.rs b/src/stream/src/executor/backfill/snapshot_backfill.rs
index 89801a3cf4133..7234a56deca26 100644
--- a/src/stream/src/executor/backfill/snapshot_backfill.rs
+++ b/src/stream/src/executor/backfill/snapshot_backfill.rs
@@ -12,17 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::cmp::min;
 use std::collections::VecDeque;
 use std::future::{pending, Future};
-use std::mem::replace;
+use std::mem::{replace, take};
 use std::sync::Arc;
 
 use anyhow::anyhow;
 use futures::future::Either;
 use futures::{pin_mut, Stream, TryStreamExt};
-use itertools::Itertools;
 use risingwave_common::array::{Op, StreamChunk};
-use risingwave_common::catalog::TableId;
 use risingwave_common::metrics::LabelGuardedIntCounter;
 use risingwave_common::row::OwnedRow;
 use risingwave_common::util::chunk_coalesce::DataChunkBuilder;
@@ -32,7 +31,6 @@ use risingwave_storage::table::batch_table::storage_table::StorageTable;
 use risingwave_storage::table::ChangeLogRow;
 use risingwave_storage::StateStore;
 use tokio::select;
-use tokio::sync::mpsc;
 use tokio::sync::mpsc::UnboundedReceiver;
 
 use crate::executor::backfill::utils::{create_builder, mapping_chunk};
@@ -40,7 +38,7 @@ use crate::executor::monitor::StreamingMetrics;
 use crate::executor::prelude::{try_stream, StreamExt};
 use crate::executor::{
     expect_first_barrier, ActorContextRef, BackfillExecutor, Barrier, BoxedMessageStream,
-    DispatcherBarrier, DispatcherMessage, Execute, MergeExecutorInput, Message, Mutation,
+    DispatcherBarrier, DispatcherMessage, Execute, MergeExecutorInput, Message,
     StreamExecutorError, StreamExecutorResult,
 };
 use crate::task::CreateMviewProgressReporter;
@@ -101,35 +99,14 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute_inner(mut self) {
         debug!("snapshot backfill executor start");
-        let upstream_table_id = self.upstream_table.table_id();
         let first_barrier = expect_first_barrier(&mut self.upstream).await?;
         debug!(epoch = ?first_barrier.epoch, "get first upstream barrier");
         let first_recv_barrier = receive_next_barrier(&mut self.barrier_rx).await?;
         debug!(epoch = ?first_recv_barrier.epoch, "get first inject barrier");
         let should_backfill = first_barrier.epoch != first_recv_barrier.epoch;
 
-        let mut barrier_epoch = {
+        let (mut barrier_epoch, mut need_report_finish) = {
             if should_backfill {
-                let subscriber_ids = first_recv_barrier
-                    .added_subscriber_on_mv_table(upstream_table_id)
-                    .collect_vec();
-                let snapshot_backfill_table_fragment_id = match subscriber_ids.as_slice() {
-                    [] => {
-                        return Err(anyhow!(
-                            "first recv barrier on backfill should add subscriber on upstream"
-                        )
-                        .into());
-                    }
-                    [snapshot_backfill_table_fragment_id] => *snapshot_backfill_table_fragment_id,
-                    multiple => {
-                        return Err(anyhow!(
-                            "first recv barrier on backfill have multiple subscribers {:?} on upstream table {}",
-                            multiple, upstream_table_id.table_id
-                        )
-                        .into());
-                    }
-                };
-
                 let table_id_str = format!("{}", self.upstream_table.table_id().table_id);
                 let actor_id_str = format!("{}", self.actor_ctx.id);
 
@@ -138,12 +115,8 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
                     .snapshot_backfill_consume_row_count
                     .with_guarded_label_values(&[&table_id_str, &actor_id_str, "consume_upstream"]);
 
-                let mut upstream_buffer = UpstreamBuffer::new(
-                    &mut self.upstream,
-                    upstream_table_id,
-                    snapshot_backfill_table_fragment_id,
-                    consume_upstream_row_count,
-                );
+                let mut upstream_buffer =
+                    UpstreamBuffer::new(&mut self.upstream, consume_upstream_row_count);
 
                 let first_barrier_epoch = first_barrier.epoch;
 
@@ -165,7 +138,7 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
                             self.rate_limit,
                             &mut self.barrier_rx,
                             &self.output_indices,
-                            self.progress,
+                            &mut self.progress,
                             first_recv_barrier,
                         );
 
@@ -187,12 +160,11 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
                     yield Message::Barrier(recv_barrier);
                 }
 
-                let mut upstream_buffer =
-                    upstream_buffer.start_consuming_log_store(&mut self.barrier_rx);
+                let mut upstream_buffer = upstream_buffer.start_consuming_log_store();
 
                 let mut barrier_epoch = first_barrier_epoch;
 
-                let initial_pending_barrier = upstream_buffer.state.barrier_count();
+                let initial_pending_barrier = upstream_buffer.barrier_count();
                 info!(
                     ?barrier_epoch,
                     table_id = self.upstream_table.table_id().table_id,
@@ -210,37 +182,50 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
                     ]);
 
                 // Phase 2: consume upstream log store
-                while let Some(barrier) = upstream_buffer.take_buffered_barrier().await? {
-                    assert_eq!(barrier_epoch.curr, barrier.epoch.prev);
-                    barrier_epoch = barrier.epoch;
-
-                    debug!(?barrier_epoch, kind = ?barrier.kind, "before consume change log");
-                    // use `upstream_buffer.run_future` to poll upstream concurrently so that we won't have back-pressure
-                    // on the upstream. Otherwise, in `batch_iter_log_with_pk_bounds`, we may wait upstream epoch to be committed,
-                    // and the back-pressure may cause the upstream unable to consume the barrier and then cause deadlock.
-                    let stream = upstream_buffer
-                        .run_future(self.upstream_table.batch_iter_log_with_pk_bounds(
-                            barrier_epoch.prev,
-                            HummockReadEpoch::Committed(barrier_epoch.prev),
-                        ))
-                        .await?;
-                    let data_types = self.upstream_table.schema().data_types();
-                    let builder = create_builder(None, self.chunk_size, data_types);
-                    let stream = read_change_log(stream, builder);
-                    pin_mut!(stream);
-                    while let Some(chunk) = upstream_buffer.run_future(stream.try_next()).await? {
-                        debug!(
-                            ?barrier_epoch,
-                            size = chunk.cardinality(),
-                            "consume change log yield chunk",
-                        );
-                        consuming_log_store_row_count.inc_by(chunk.cardinality() as _);
-                        yield Message::Chunk(chunk);
-                    }
+                while let Some(upstream_barriers) =
+                    upstream_buffer.next_checkpoint_barrier().await?
+                {
+                    for upstream_barrier in upstream_barriers {
+                        let barrier = receive_next_barrier(&mut self.barrier_rx).await?;
+                        assert_eq!(upstream_barrier.epoch, barrier.epoch);
+                        assert_eq!(barrier_epoch.curr, barrier.epoch.prev);
+                        barrier_epoch = barrier.epoch;
+                        debug!(?barrier_epoch, kind = ?barrier.kind, "before consume change log");
+                        // use `upstream_buffer.run_future` to poll upstream concurrently so that we won't have back-pressure
+                        // on the upstream. Otherwise, in `batch_iter_log_with_pk_bounds`, we may wait upstream epoch to be committed,
+                        // and the back-pressure may cause the upstream unable to consume the barrier and then cause deadlock.
+                        let stream = upstream_buffer
+                            .run_future(self.upstream_table.batch_iter_log_with_pk_bounds(
+                                barrier_epoch.prev,
+                                HummockReadEpoch::Committed(barrier_epoch.prev),
+                                false,
+                            ))
+                            .await?;
+                        let data_types = self.upstream_table.schema().data_types();
+                        let builder = create_builder(None, self.chunk_size, data_types);
+                        let stream = read_change_log(stream, builder);
+                        pin_mut!(stream);
+                        while let Some(chunk) =
+                            upstream_buffer.run_future(stream.try_next()).await?
+                        {
+                            debug!(
+                                ?barrier_epoch,
+                                size = chunk.cardinality(),
+                                "consume change log yield chunk",
+                            );
+                            consuming_log_store_row_count.inc_by(chunk.cardinality() as _);
+                            yield Message::Chunk(chunk);
+                        }
+
+                        debug!(?barrier_epoch, "after consume change log");
 
-                    debug!(?barrier_epoch, "after consume change log");
+                        self.progress.update_create_mview_log_store_progress(
+                            barrier.epoch,
+                            upstream_buffer.barrier_count(),
+                        );
 
-                    yield Message::Barrier(barrier);
+                        yield Message::Barrier(barrier);
+                    }
                 }
 
                 info!(
@@ -248,7 +233,7 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
                     table_id = self.upstream_table.table_id().table_id,
                     "finish consuming log store"
                 );
-                barrier_epoch
+                (barrier_epoch, true)
             } else {
                 info!(
                     table_id = self.upstream_table.table_id().table_id,
@@ -257,7 +242,7 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
                 let first_recv_barrier = receive_next_barrier(&mut self.barrier_rx).await?;
                 assert_eq!(first_barrier.epoch, first_recv_barrier.epoch);
                 yield Message::Barrier(first_recv_barrier);
-                first_barrier.epoch
+                (first_barrier.epoch, false)
             }
         };
         let mut upstream = self.upstream.into_executor(self.barrier_rx).execute();
@@ -266,6 +251,10 @@ impl<S: StateStore> SnapshotBackfillExecutor<S> {
             if let Message::Barrier(barrier) = &msg {
                 assert_eq!(barrier.epoch.prev, barrier_epoch.curr);
                 barrier_epoch = barrier.epoch;
+                if need_report_finish {
+                    need_report_finish = false;
+                    self.progress.finish_consuming_log_store(barrier_epoch);
+                }
             }
             yield msg;
         }
@@ -328,146 +317,84 @@ async fn read_change_log(
     }
 }
 
-trait UpstreamBufferState {
-    // The future must be cancellation-safe
-    async fn is_finished(&mut self) -> StreamExecutorResult<bool>;
-    fn on_upstream_barrier(&mut self, upstream_barrier: DispatcherBarrier);
-}
-
-struct StateOfConsumingSnapshot {
-    pending_barriers: Vec<DispatcherBarrier>,
-}
-
-impl UpstreamBufferState for StateOfConsumingSnapshot {
-    async fn is_finished(&mut self) -> StreamExecutorResult<bool> {
-        // never finish when consuming snapshot
-        Ok(false)
-    }
-
-    fn on_upstream_barrier(&mut self, upstream_barrier: DispatcherBarrier) {
-        self.pending_barriers.push(upstream_barrier)
-    }
-}
-
-struct StateOfConsumingLogStore<'a> {
-    barrier_rx: &'a mut mpsc::UnboundedReceiver<Barrier>,
-    /// Barriers received from upstream but not yet received the barrier from local barrier worker
-    /// newer barrier at the front
-    upstream_pending_barriers: VecDeque<DispatcherBarrier>,
-    /// Barriers received from both upstream and local barrier worker
-    /// newer barrier at the front
-    barriers: VecDeque<Barrier>,
-    is_finished: bool,
-    current_subscriber_id: u32,
-    upstream_table_id: TableId,
-}
-
-impl<'a> StateOfConsumingLogStore<'a> {
-    fn barrier_count(&self) -> usize {
-        self.upstream_pending_barriers.len() + self.barriers.len()
-    }
-
-    async fn handle_one_pending_barrier(&mut self) -> StreamExecutorResult<Barrier> {
-        assert!(!self.is_finished);
-        let barrier = receive_next_barrier(self.barrier_rx).await?;
-        assert_eq!(
-            self.upstream_pending_barriers
-                .pop_back()
-                .expect("non-empty")
-                .epoch,
-            barrier.epoch
-        );
-        if is_finish_barrier(&barrier, self.current_subscriber_id, self.upstream_table_id) {
-            self.is_finished = true;
-        }
-        Ok(barrier)
-    }
-}
-
-impl<'a> UpstreamBufferState for StateOfConsumingLogStore<'a> {
-    async fn is_finished(&mut self) -> StreamExecutorResult<bool> {
-        while !self.upstream_pending_barriers.is_empty() {
-            let barrier = self.handle_one_pending_barrier().await?;
-            self.barriers.push_front(barrier);
-        }
-        if self.is_finished {
-            assert!(self.upstream_pending_barriers.is_empty());
-        }
-        Ok(self.is_finished)
-    }
-
-    fn on_upstream_barrier(&mut self, upstream_barrier: DispatcherBarrier) {
-        self.upstream_pending_barriers.push_front(upstream_barrier);
-    }
-}
+struct ConsumingSnapshot;
+struct ConsumingLogStore;
 
 struct UpstreamBuffer<'a, S> {
     upstream: &'a mut MergeExecutorInput,
-    state: S,
+    max_pending_checkpoint_barrier_num: usize,
+    pending_non_checkpoint_barriers: Vec<DispatcherBarrier>,
+    /// Barriers received from upstream but not yet received the barrier from local barrier worker.
+    ///
+    /// In the outer `VecDeque`, newer barriers at the front.
+    /// In the inner `Vec`, newer barrier at the back, with the last barrier as checkpoint barrier,
+    /// and others as non-checkpoint barrier
+    upstream_pending_barriers: VecDeque<Vec<DispatcherBarrier>>,
+    /// Whether we have started polling any upstream data before the next barrier.
+    /// When `true`, we should continue polling until the next barrier, because
+    /// some data in this epoch have been discarded and data in this epoch
+    /// must be read from log store
+    is_polling_epoch_data: bool,
     consume_upstream_row_count: LabelGuardedIntCounter<3>,
-    upstream_table_id: TableId,
-    current_subscriber_id: u32,
+    _phase: S,
 }
 
-impl<'a> UpstreamBuffer<'a, StateOfConsumingSnapshot> {
+impl<'a> UpstreamBuffer<'a, ConsumingSnapshot> {
     fn new(
         upstream: &'a mut MergeExecutorInput,
-        upstream_table_id: TableId,
-        current_subscriber_id: u32,
         consume_upstream_row_count: LabelGuardedIntCounter<3>,
     ) -> Self {
         Self {
             upstream,
-            state: StateOfConsumingSnapshot {
-                pending_barriers: vec![],
-            },
+            is_polling_epoch_data: false,
             consume_upstream_row_count,
-            upstream_table_id,
-            current_subscriber_id,
+            pending_non_checkpoint_barriers: vec![],
+            upstream_pending_barriers: Default::default(),
+            // no limit on the number of pending barrier in the beginning
+            max_pending_checkpoint_barrier_num: usize::MAX,
+            _phase: ConsumingSnapshot {},
         }
     }
 
-    fn start_consuming_log_store<'s>(
-        self,
-        barrier_rx: &'s mut UnboundedReceiver<Barrier>,
-    ) -> UpstreamBuffer<'a, StateOfConsumingLogStore<'s>> {
-        let StateOfConsumingSnapshot { pending_barriers } = self.state;
-        let mut upstream_pending_barriers = VecDeque::with_capacity(pending_barriers.len());
-        for pending_barrier in pending_barriers {
-            upstream_pending_barriers.push_front(pending_barrier);
-        }
+    fn start_consuming_log_store(self) -> UpstreamBuffer<'a, ConsumingLogStore> {
+        let max_pending_barrier_num = self.barrier_count();
         UpstreamBuffer {
             upstream: self.upstream,
-            state: StateOfConsumingLogStore {
-                barrier_rx,
-                upstream_pending_barriers,
-                barriers: Default::default(),
-                is_finished: false,
-                current_subscriber_id: self.current_subscriber_id,
-                upstream_table_id: self.upstream_table_id,
-            },
+            pending_non_checkpoint_barriers: self.pending_non_checkpoint_barriers,
+            upstream_pending_barriers: self.upstream_pending_barriers,
+            max_pending_checkpoint_barrier_num: max_pending_barrier_num,
+            is_polling_epoch_data: self.is_polling_epoch_data,
             consume_upstream_row_count: self.consume_upstream_row_count,
-            upstream_table_id: self.upstream_table_id,
-            current_subscriber_id: self.current_subscriber_id,
+            _phase: ConsumingLogStore {},
         }
     }
 }
 
-impl<'a, S: UpstreamBufferState> UpstreamBuffer<'a, S> {
+impl<'a, S> UpstreamBuffer<'a, S> {
     async fn concurrently_consume_upstream(&mut self) -> StreamExecutorError {
-        if let Err(e) = try {
-            while !self.state.is_finished().await? {
-                self.consume_until_next_barrier().await?;
+        {
+            loop {
+                if let Err(e) = try {
+                    if self.upstream_pending_barriers.len()
+                        >= self.max_pending_checkpoint_barrier_num
+                    {
+                        // pause the future to block consuming upstream
+                        return pending().await;
+                    }
+                    let barrier = self.consume_until_next_checkpoint_barrier().await?;
+                    self.upstream_pending_barriers.push_front(barrier);
+                } {
+                    break e;
+                }
             }
-        } {
-            return e;
         }
-        pending().await
     }
 
     /// Consume the upstream until seeing the next barrier.
     /// `pending_barriers` must be non-empty after this method returns.
-    async fn consume_until_next_barrier(&mut self) -> StreamExecutorResult<()> {
+    async fn consume_until_next_checkpoint_barrier(
+        &mut self,
+    ) -> StreamExecutorResult<Vec<DispatcherBarrier>> {
         loop {
             let msg: DispatcherMessage = self
                 .upstream
@@ -476,63 +403,54 @@ impl<'a, S: UpstreamBufferState> UpstreamBuffer<'a, S> {
                 .ok_or_else(|| anyhow!("end of upstream"))?;
             match msg {
                 DispatcherMessage::Chunk(chunk) => {
+                    self.is_polling_epoch_data = true;
                     self.consume_upstream_row_count
                         .inc_by(chunk.cardinality() as _);
                 }
                 DispatcherMessage::Barrier(barrier) => {
-                    self.state.on_upstream_barrier(barrier);
-                    break Ok(());
+                    let is_checkpoint = barrier.kind.is_checkpoint();
+                    self.pending_non_checkpoint_barriers.push(barrier);
+                    if is_checkpoint {
+                        self.is_polling_epoch_data = false;
+                        break Ok(take(&mut self.pending_non_checkpoint_barriers));
+                    } else {
+                        self.is_polling_epoch_data = true;
+                    }
+                }
+                DispatcherMessage::Watermark(_) => {
+                    self.is_polling_epoch_data = true;
                 }
-                DispatcherMessage::Watermark(_) => {}
             }
         }
     }
 }
 
-impl<'a, 's> UpstreamBuffer<'a, StateOfConsumingLogStore<'s>> {
-    async fn take_buffered_barrier(&mut self) -> StreamExecutorResult<Option<Barrier>> {
-        Ok(if let Some(barrier) = self.state.barriers.pop_back() {
-            Some(barrier)
-        } else if !self.state.upstream_pending_barriers.is_empty() {
-            let barrier = self.state.handle_one_pending_barrier().await?;
-            Some(barrier)
-        } else if self.state.is_finished {
-            None
-        } else {
-            self.consume_until_next_barrier().await?;
-            let barrier = self.state.handle_one_pending_barrier().await?;
-            Some(barrier)
-        })
-    }
-}
-
-fn is_finish_barrier(
-    barrier: &Barrier,
-    current_subscriber_id: u32,
-    upstream_table_id: TableId,
-) -> bool {
-    if let Some(Mutation::DropSubscriptions {
-        subscriptions_to_drop,
-    }) = barrier.mutation.as_deref()
-    {
-        let is_finished = subscriptions_to_drop
-            .iter()
-            .any(|(subscriber_id, _)| *subscriber_id == current_subscriber_id);
-        if is_finished {
-            assert!(subscriptions_to_drop.iter().any(
-                |(subscriber_id, subscribed_upstream_table_id)| {
-                    *subscriber_id == current_subscriber_id
-                        && upstream_table_id == *subscribed_upstream_table_id
+impl<'a> UpstreamBuffer<'a, ConsumingLogStore> {
+    async fn next_checkpoint_barrier(
+        &mut self,
+    ) -> StreamExecutorResult<Option<Vec<DispatcherBarrier>>> {
+        Ok(
+            if let Some(barriers) = self.upstream_pending_barriers.pop_back() {
+                // sub(1) to ensure that the lag is monotonically decreasing.
+                self.max_pending_checkpoint_barrier_num = min(
+                    self.upstream_pending_barriers.len(),
+                    self.max_pending_checkpoint_barrier_num.saturating_sub(1),
+                );
+                Some(barriers)
+            } else {
+                self.max_pending_checkpoint_barrier_num = 0;
+                if self.is_polling_epoch_data {
+                    let barriers = self.consume_until_next_checkpoint_barrier().await?;
+                    Some(barriers)
+                } else {
+                    None
                 }
-            ))
-        }
-        is_finished
-    } else {
-        false
+            },
+        )
     }
 }
 
-impl<'a, S: UpstreamBufferState> UpstreamBuffer<'a, S> {
+impl<'a, S> UpstreamBuffer<'a, S> {
     /// Run a future while concurrently polling the upstream so that the upstream
     /// won't be back-pressured.
     async fn run_future<T, E: Into<StreamExecutorError>>(
@@ -550,6 +468,10 @@ impl<'a, S: UpstreamBufferState> UpstreamBuffer<'a, S> {
             }
         }
     }
+
+    fn barrier_count(&self) -> usize {
+        self.upstream_pending_barriers.len()
+    }
 }
 
 async fn receive_next_barrier(
@@ -589,7 +511,7 @@ async fn make_consume_snapshot_stream<'a, S: StateStore>(
     rate_limit: Option<usize>,
     barrier_rx: &'a mut UnboundedReceiver<Barrier>,
     output_indices: &'a [usize],
-    mut progress: CreateMviewProgressReporter,
+    progress: &'a mut CreateMviewProgressReporter,
     first_recv_barrier: Barrier,
 ) {
     let mut barrier_epoch = first_recv_barrier.epoch;
diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs
index eedba01799c65..25003aff7f70b 100644
--- a/src/stream/src/executor/hash_agg.rs
+++ b/src/stream/src/executor/hash_agg.rs
@@ -286,7 +286,7 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
                         Some(async {
                             // Create `AggGroup` for the current group if not exists. This will
                             // restore agg states from the intermediate state table.
-                            let (agg_group, stats) = AggGroup::create(
+                            let agg_group = AggGroup::create(
                                 this.version,
                                 Some(GroupKey::new(
                                     key.deserialize(group_key_types)?,
@@ -302,7 +302,7 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
                                 &this.input_schema,
                             )
                             .await?;
-                            Ok::<_, StreamExecutorError>((key.clone(), Box::new(agg_group), stats))
+                            Ok::<_, StreamExecutorError>((key.clone(), Box::new(agg_group)))
                         })
                     }
                 }
@@ -315,10 +315,9 @@ impl<K: HashKey, S: StateStore> HashAggExecutor<K, S> {
             vars.stats.chunk_lookup_miss_count += 1;
             let mut buffered = stream::iter(futs).buffer_unordered(10).fuse();
             while let Some(result) = buffered.next().await {
-                let (key, agg_group, stats) = result?;
+                let (key, agg_group) = result?;
                 let none = vars.dirty_groups.insert(key, agg_group);
                 debug_assert!(none.is_none());
-                vars.stats.merge_state_cache_stats(stats);
             }
         }
         Ok(())
diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs
index 6ea34de857219..41495198632f9 100644
--- a/src/stream/src/executor/integration_tests.rs
+++ b/src/stream/src/executor/integration_tests.rs
@@ -176,17 +176,19 @@ async fn test_merger_sum_aggr() {
         let items = items.clone();
         async move {
             // use a merge operator to collect data from dispatchers before sending them to aggregator
+            let schema = Schema::new(vec![
+                Field::unnamed(DataType::Int64),
+                Field::unnamed(DataType::Int64),
+            ]);
             let merger = Executor::new(
                 ExecutorInfo {
                     // output schema of local simple agg
-                    schema: Schema::new(vec![
-                        Field::unnamed(DataType::Int64),
-                        Field::unnamed(DataType::Int64),
-                    ]),
+                    schema: schema.clone(),
                     pk_indices: PkIndices::new(),
                     identity: "MergeExecutor".to_string(),
                 },
-                MergeExecutor::for_test(actor_ctx.id, outputs, shared_context.clone()).boxed(),
+                MergeExecutor::for_test(actor_ctx.id, outputs, shared_context.clone(), schema)
+                    .boxed(),
             );
 
             // for global aggregator, we need to sum data and sum row count
@@ -217,7 +219,6 @@ async fn test_merger_sum_aggr() {
                 ],
                 MultiMap::new(),
                 vec![],
-                0.0,
                 false,
             );
 
diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs
index 0316f7cf36796..8136662b5eaf9 100644
--- a/src/stream/src/executor/merge.rs
+++ b/src/stream/src/executor/merge.rs
@@ -12,13 +12,14 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, VecDeque};
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
 use anyhow::Context as _;
 use futures::stream::{FusedStream, FuturesUnordered, StreamFuture};
 use prometheus::Histogram;
+use risingwave_common::array::StreamChunkBuilder;
 use risingwave_common::config::MetricLevel;
 use risingwave_common::metrics::LabelGuardedMetric;
 use tokio::sync::mpsc;
@@ -45,6 +46,7 @@ pub(crate) struct MergeExecutorInput {
     shared_context: Arc<SharedContext>,
     executor_stats: Arc<StreamingMetrics>,
     info: ExecutorInfo,
+    chunk_size: usize,
 }
 
 impl MergeExecutorInput {
@@ -55,6 +57,7 @@ impl MergeExecutorInput {
         shared_context: Arc<SharedContext>,
         executor_stats: Arc<StreamingMetrics>,
         info: ExecutorInfo,
+        chunk_size: usize,
     ) -> Self {
         Self {
             upstream,
@@ -63,6 +66,7 @@ impl MergeExecutorInput {
             shared_context,
             executor_stats,
             info,
+            chunk_size,
         }
     }
 
@@ -87,6 +91,8 @@ impl MergeExecutorInput {
                 self.shared_context,
                 self.executor_stats,
                 barrier_rx,
+                self.chunk_size,
+                self.info.schema.clone(),
             )
             .boxed(),
         };
@@ -127,6 +133,12 @@ pub struct MergeExecutor {
     metrics: Arc<StreamingMetrics>,
 
     barrier_rx: mpsc::UnboundedReceiver<Barrier>,
+
+    /// Chunk size for the `StreamChunkBuilder`
+    chunk_size: usize,
+
+    /// Data types for the `StreamChunkBuilder`
+    schema: Schema,
 }
 
 impl MergeExecutor {
@@ -139,6 +151,8 @@ impl MergeExecutor {
         context: Arc<SharedContext>,
         metrics: Arc<StreamingMetrics>,
         barrier_rx: mpsc::UnboundedReceiver<Barrier>,
+        chunk_size: usize,
+        schema: Schema,
     ) -> Self {
         Self {
             actor_context: ctx,
@@ -148,6 +162,8 @@ impl MergeExecutor {
             context,
             metrics,
             barrier_rx,
+            chunk_size,
+            schema,
         }
     }
 
@@ -156,6 +172,7 @@ impl MergeExecutor {
         actor_id: ActorId,
         inputs: Vec<super::exchange::permit::Receiver>,
         shared_context: Arc<SharedContext>,
+        schema: Schema,
     ) -> Self {
         use super::exchange::input::LocalInput;
         use crate::executor::exchange::input::Input;
@@ -184,6 +201,8 @@ impl MergeExecutor {
             shared_context,
             metrics.into(),
             barrier_rx,
+            100,
+            schema,
         )
     }
 
@@ -216,6 +235,7 @@ impl MergeExecutor {
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute_inner(mut self: Box<Self>) {
         let select_all = self.upstreams;
+        let select_all = BufferChunks::new(select_all, self.chunk_size, self.schema);
         let actor_id = self.actor_context.id;
 
         let mut metrics = self.metrics.new_actor_input_metrics(
@@ -560,6 +580,91 @@ impl SelectReceivers {
     }
 }
 
+/// A wrapper that buffers the `StreamChunk`s from upstream until no more ready items are available.
+/// Besides, any message other than `StreamChunk` will trigger the buffered `StreamChunk`s
+/// to be emitted immediately along with the message itself.
+struct BufferChunks<S: Stream> {
+    inner: S,
+    chunk_builder: StreamChunkBuilder,
+
+    /// The items to be emitted. Whenever there's something here, we should return a `Poll::Ready` immediately.
+    pending_items: VecDeque<S::Item>,
+}
+
+impl<S: Stream> BufferChunks<S> {
+    pub(super) fn new(inner: S, chunk_size: usize, schema: Schema) -> Self {
+        assert!(chunk_size > 0);
+        let chunk_builder = StreamChunkBuilder::new(chunk_size, schema.data_types());
+        Self {
+            inner,
+            chunk_builder,
+            pending_items: VecDeque::new(),
+        }
+    }
+}
+
+impl<S: Stream> std::ops::Deref for BufferChunks<S> {
+    type Target = S;
+
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl<S: Stream> std::ops::DerefMut for BufferChunks<S> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
+    }
+}
+
+impl<S: Stream> Stream for BufferChunks<S>
+where
+    S: Stream<Item = DispatcherMessageStreamItem> + Unpin,
+{
+    type Item = S::Item;
+
+    fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
+        loop {
+            if let Some(item) = self.pending_items.pop_front() {
+                return Poll::Ready(Some(item));
+            }
+
+            match self.inner.poll_next_unpin(cx) {
+                Poll::Pending => {
+                    return if let Some(chunk_out) = self.chunk_builder.take() {
+                        Poll::Ready(Some(Ok(MessageInner::Chunk(chunk_out))))
+                    } else {
+                        Poll::Pending
+                    }
+                }
+
+                Poll::Ready(Some(result)) => {
+                    if let Ok(MessageInner::Chunk(chunk)) = result {
+                        for row in chunk.records() {
+                            if let Some(chunk_out) = self.chunk_builder.append_record(row) {
+                                self.pending_items
+                                    .push_back(Ok(MessageInner::Chunk(chunk_out)));
+                            }
+                        }
+                    } else {
+                        return if let Some(chunk_out) = self.chunk_builder.take() {
+                            self.pending_items.push_back(result);
+                            Poll::Ready(Some(Ok(MessageInner::Chunk(chunk_out))))
+                        } else {
+                            Poll::Ready(Some(result))
+                        };
+                    }
+                }
+
+                Poll::Ready(None) => {
+                    // See also the comments in `SelectReceivers::poll_next`.
+                    unreachable!("SelectReceivers should never return None");
+                }
+            }
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::sync::atomic::{AtomicBool, Ordering};
@@ -582,18 +687,93 @@ mod tests {
     use tonic::{Request, Response, Status, Streaming};
 
     use super::*;
-    use crate::executor::exchange::input::{Input, RemoteInput};
+    use crate::executor::exchange::input::{Input, LocalInput, RemoteInput};
     use crate::executor::exchange::permit::channel_for_test;
     use crate::executor::{BarrierInner as Barrier, MessageInner as Message};
     use crate::task::barrier_test_utils::LocalBarrierTestEnv;
     use crate::task::test_utils::helper_make_local_actor;
 
-    fn build_test_chunk(epoch: u64) -> StreamChunk {
-        // The number of items in `ops` is the epoch count.
-        let ops = vec![Op::Insert; epoch as usize];
+    fn build_test_chunk(size: u64) -> StreamChunk {
+        let ops = vec![Op::Insert; size as usize];
         StreamChunk::new(ops, vec![])
     }
 
+    #[tokio::test]
+    async fn test_buffer_chunks() {
+        let test_env = LocalBarrierTestEnv::for_test().await;
+
+        let (tx, rx) = channel_for_test();
+        let input = LocalInput::new(rx, 1).boxed_input();
+        let mut buffer = BufferChunks::new(input, 100, Schema::new(vec![]));
+
+        // Send a chunk
+        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Chunk(chunk) => {
+            assert_eq!(chunk.ops().len() as u64, 10);
+        });
+
+        // Send 2 chunks and expect them to be merged.
+        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
+        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Chunk(chunk) => {
+            assert_eq!(chunk.ops().len() as u64, 20);
+        });
+
+        // Send a watermark.
+        tx.send(Message::Watermark(Watermark {
+            col_idx: 0,
+            data_type: DataType::Int64,
+            val: ScalarImpl::Int64(233),
+        }))
+        .await
+        .unwrap();
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Watermark(watermark) => {
+            assert_eq!(watermark.val, ScalarImpl::Int64(233));
+        });
+
+        // Send 2 chunks before a watermark. Expect the 2 chunks to be merged and the watermark to be emitted.
+        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
+        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
+        tx.send(Message::Watermark(Watermark {
+            col_idx: 0,
+            data_type: DataType::Int64,
+            val: ScalarImpl::Int64(233),
+        }))
+        .await
+        .unwrap();
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Chunk(chunk) => {
+            assert_eq!(chunk.ops().len() as u64, 20);
+        });
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Watermark(watermark) => {
+            assert_eq!(watermark.val, ScalarImpl::Int64(233));
+        });
+
+        // Send a barrier.
+        let barrier = Barrier::new_test_barrier(test_epoch(1));
+        test_env.inject_barrier(&barrier, [2]);
+        tx.send(Message::Barrier(barrier.clone().into_dispatcher()))
+            .await
+            .unwrap();
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Barrier(Barrier { epoch: barrier_epoch, mutation: _, .. }) => {
+            assert_eq!(barrier_epoch.curr, test_epoch(1));
+        });
+
+        // Send 2 chunks before a barrier. Expect the 2 chunks to be merged and the barrier to be emitted.
+        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
+        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
+        let barrier = Barrier::new_test_barrier(test_epoch(2));
+        test_env.inject_barrier(&barrier, [2]);
+        tx.send(Message::Barrier(barrier.clone().into_dispatcher()))
+            .await
+            .unwrap();
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Chunk(chunk) => {
+            assert_eq!(chunk.ops().len() as u64, 20);
+        });
+        assert_matches!(buffer.next().await.unwrap().unwrap(), Message::Barrier(Barrier { epoch: barrier_epoch, mutation: _, .. }) => {
+            assert_eq!(barrier_epoch.curr, test_epoch(2));
+        });
+    }
+
     #[tokio::test]
     async fn test_merger() {
         const CHANNEL_NUMBER: usize = 10;
@@ -639,9 +819,7 @@ mod tests {
             let handle = tokio::spawn(async move {
                 for (idx, epoch) in epochs {
                     if idx % 20 == 0 {
-                        tx.send(Message::Chunk(build_test_chunk(idx)))
-                            .await
-                            .unwrap();
+                        tx.send(Message::Chunk(build_test_chunk(10))).await.unwrap();
                     } else {
                         tx.send(Message::Watermark(Watermark {
                             col_idx: (idx as usize / 20 + tx_id) % CHANNEL_NUMBER,
@@ -663,18 +841,25 @@ mod tests {
             handles.push(handle);
         }
 
-        let merger =
-            MergeExecutor::for_test(actor_id, rxs, barrier_test_env.shared_context.clone());
+        let merger = MergeExecutor::for_test(
+            actor_id,
+            rxs,
+            barrier_test_env.shared_context.clone(),
+            Schema::new(vec![]),
+        );
         let mut merger = merger.boxed().execute();
         for (idx, epoch) in epochs {
-            // expect n chunks
             if idx % 20 == 0 {
-                for _ in 0..CHANNEL_NUMBER {
+                // expect 1 or more chunks with 100 rows in total
+                let mut count = 0usize;
+                while count < 100 {
                     assert_matches!(merger.next().await.unwrap().unwrap(), Message::Chunk(chunk) => {
-                        assert_eq!(chunk.ops().len() as u64, idx);
+                        count += chunk.ops().len();
                     });
                 }
+                assert_eq!(count, 100);
             } else if idx as usize / 20 >= CHANNEL_NUMBER - 1 {
+                // expect n watermarks
                 for _ in 0..CHANNEL_NUMBER {
                     assert_matches!(merger.next().await.unwrap().unwrap(), Message::Watermark(watermark) => {
                         assert_eq!(watermark.val, ScalarImpl::Int64((idx - 20 * (CHANNEL_NUMBER as u64 - 1)) as i64));
@@ -775,6 +960,8 @@ mod tests {
             ctx.clone(),
             metrics.clone(),
             barrier_rx,
+            100,
+            Schema::new(vec![]),
         )
         .boxed()
         .execute();
@@ -810,9 +997,8 @@ mod tests {
         }
 
         // 3. Send a chunk.
-        send!([untouched, old], Message::Chunk(StreamChunk::default()));
-        recv!().unwrap().as_chunk().unwrap(); // We should be able to receive the chunk twice.
-        recv!().unwrap().as_chunk().unwrap();
+        send!([untouched, old], Message::Chunk(build_test_chunk(1)));
+        assert_eq!(2, recv!().unwrap().as_chunk().unwrap().cardinality()); // We should be able to receive the chunk twice.
         assert_recv_pending!();
 
         send!(
@@ -825,9 +1011,8 @@ mod tests {
         recv!().unwrap().as_barrier().unwrap(); // We should now receive the barrier.
 
         // 5. Send a chunk.
-        send!([untouched, new], Message::Chunk(StreamChunk::default()));
-        recv!().unwrap().as_chunk().unwrap(); // We should be able to receive the chunk twice, since old is removed.
-        recv!().unwrap().as_chunk().unwrap();
+        send!([untouched, new], Message::Chunk(build_test_chunk(1)));
+        assert_eq!(2, recv!().unwrap().as_chunk().unwrap().cardinality()); // We should be able to receive the chunk twice.
         assert_recv_pending!();
     }
 
diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs
index a053e7dc50213..0e5e7862e9605 100644
--- a/src/stream/src/executor/mod.rs
+++ b/src/stream/src/executor/mod.rs
@@ -140,6 +140,7 @@ pub use lookup_union::LookupUnionExecutor;
 pub use merge::MergeExecutor;
 pub(crate) use merge::{MergeExecutorInput, MergeExecutorUpstream};
 pub use mview::*;
+pub use nested_loop_temporal_join::NestedLoopTemporalJoinExecutor;
 pub use no_op::NoOpExecutor;
 pub use now::*;
 pub use over_window::*;
diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs
index cc8473d99ae21..6119882297a9a 100644
--- a/src/stream/src/executor/monitor/streaming_stats.rs
+++ b/src/stream/src/executor/monitor/streaming_stats.rs
@@ -150,8 +150,9 @@ pub struct StreamingMetrics {
     over_window_range_cache_lookup_count: LabelGuardedIntCounterVec<3>,
     over_window_range_cache_left_miss_count: LabelGuardedIntCounterVec<3>,
     over_window_range_cache_right_miss_count: LabelGuardedIntCounterVec<3>,
+    over_window_accessed_entry_count: LabelGuardedIntCounterVec<3>,
     over_window_compute_count: LabelGuardedIntCounterVec<3>,
-    over_window_same_result_count: LabelGuardedIntCounterVec<3>,
+    over_window_same_output_count: LabelGuardedIntCounterVec<3>,
 
     /// The duration from receipt of barrier to all actors collection.
     /// And the max of all node `barrier_inflight_latency` is the latency for a barrier
@@ -790,6 +791,14 @@ impl StreamingMetrics {
             )
             .unwrap();
 
+        let over_window_accessed_entry_count = register_guarded_int_counter_vec_with_registry!(
+            "stream_over_window_accessed_entry_count",
+            "Over window accessed entry count",
+            &["table_id", "actor_id", "fragment_id"],
+            registry
+        )
+        .unwrap();
+
         let over_window_compute_count = register_guarded_int_counter_vec_with_registry!(
             "stream_over_window_compute_count",
             "Over window compute count",
@@ -798,9 +807,9 @@ impl StreamingMetrics {
         )
         .unwrap();
 
-        let over_window_same_result_count = register_guarded_int_counter_vec_with_registry!(
-            "stream_over_window_same_result_count",
-            "Over window same result count",
+        let over_window_same_output_count = register_guarded_int_counter_vec_with_registry!(
+            "stream_over_window_same_output_count",
+            "Over window same output count",
             &["table_id", "actor_id", "fragment_id"],
             registry
         )
@@ -1096,8 +1105,9 @@ impl StreamingMetrics {
             over_window_range_cache_lookup_count,
             over_window_range_cache_left_miss_count,
             over_window_range_cache_right_miss_count,
+            over_window_accessed_entry_count,
             over_window_compute_count,
-            over_window_same_result_count,
+            over_window_same_output_count,
             barrier_inflight_latency,
             barrier_sync_latency,
             barrier_manager_progress,
@@ -1451,11 +1461,14 @@ impl StreamingMetrics {
             over_window_range_cache_right_miss_count: self
                 .over_window_range_cache_right_miss_count
                 .with_guarded_label_values(label_list),
+            over_window_accessed_entry_count: self
+                .over_window_accessed_entry_count
+                .with_guarded_label_values(label_list),
             over_window_compute_count: self
                 .over_window_compute_count
                 .with_guarded_label_values(label_list),
-            over_window_same_result_count: self
-                .over_window_same_result_count
+            over_window_same_output_count: self
+                .over_window_same_output_count
                 .with_guarded_label_values(label_list),
         }
     }
@@ -1570,6 +1583,7 @@ pub struct OverWindowMetrics {
     pub over_window_range_cache_lookup_count: LabelGuardedIntCounter<3>,
     pub over_window_range_cache_left_miss_count: LabelGuardedIntCounter<3>,
     pub over_window_range_cache_right_miss_count: LabelGuardedIntCounter<3>,
+    pub over_window_accessed_entry_count: LabelGuardedIntCounter<3>,
     pub over_window_compute_count: LabelGuardedIntCounter<3>,
-    pub over_window_same_result_count: LabelGuardedIntCounter<3>,
+    pub over_window_same_output_count: LabelGuardedIntCounter<3>,
 }
diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs
index a5dc24d5cd74b..c2cbef8e6114e 100644
--- a/src/stream/src/executor/mview/materialize.rs
+++ b/src/stream/src/executor/mview/materialize.rs
@@ -702,11 +702,9 @@ impl<SD: ValueRowSerde> MaterializeCache<SD> {
                         ConflictBehavior::Overwrite
                         | ConflictBehavior::IgnoreConflict
                         | ConflictBehavior::DoUpdateIfNotNull => {
-                            match self.force_get(&key) {
-                                Some(old_row) => {
-                                    fixed_changes().delete(key.clone(), old_row.row.clone());
-                                }
-                                None => (), // delete a nonexistent value
+                            // delete a nonexistent value
+                            if let Some(old_row) = self.force_get(&key) {
+                                fixed_changes().delete(key.clone(), old_row.row.clone());
                             };
                         }
                         _ => unreachable!(),
diff --git a/src/stream/src/executor/over_window/frame_finder.rs b/src/stream/src/executor/over_window/frame_finder.rs
index 12e9f21842887..9cffb6e26d903 100644
--- a/src/stream/src/executor/over_window/frame_finder.rs
+++ b/src/stream/src/executor/over_window/frame_finder.rs
@@ -17,7 +17,7 @@
 
 use std::ops::Bound;
 
-use delta_btree_map::DeltaBTreeMap;
+use delta_btree_map::{CursorWithDelta, DeltaBTreeMap};
 use itertools::Itertools;
 use risingwave_common::row::OwnedRow;
 use risingwave_common::types::{Datum, Sentinelled, ToDatumRef};
@@ -150,7 +150,7 @@ pub(super) fn find_frame_end_for_rows_frame<'cache>(
 /// Given the first and last key in delta, calculate the order values of the first
 /// and the last frames logically affected by some `RANGE` frames.
 pub(super) fn calc_logical_curr_for_range_frames(
-    range_frames: &[&RangeFrameBounds],
+    range_frames: &[RangeFrameBounds],
     delta_first_key: &StateKey,
     delta_last_key: &StateKey,
 ) -> Option<(Sentinelled<Datum>, Sentinelled<Datum>)> {
@@ -167,7 +167,7 @@ pub(super) fn calc_logical_curr_for_range_frames(
 /// values of the logical start row of the first frame and the logical end row of the
 /// last frame.
 pub(super) fn calc_logical_boundary_for_range_frames(
-    range_frames: &[&RangeFrameBounds],
+    range_frames: &[RangeFrameBounds],
     first_curr_key: &StateKey,
     last_curr_key: &StateKey,
 ) -> Option<(Sentinelled<Datum>, Sentinelled<Datum>)> {
@@ -184,7 +184,7 @@ pub(super) fn calc_logical_boundary_for_range_frames(
 /// find the most closed cache key in `part_with_delta`. Ideally this function returns
 /// the smallest key that is larger than or equal to the given logical order (using `lower_bound`).
 pub(super) fn find_left_for_range_frames<'cache>(
-    range_frames: &[&RangeFrameBounds],
+    range_frames: &[RangeFrameBounds],
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     logical_order_value: impl ToDatumRef,
     cache_key_pk_len: usize, // this is dirty but we have no better choice
@@ -201,7 +201,7 @@ pub(super) fn find_left_for_range_frames<'cache>(
 /// find the most closed cache key in `part_with_delta`. Ideally this function returns
 /// the largest key that is smaller than or equal to the given logical order (using `lower_bound`).
 pub(super) fn find_right_for_range_frames<'cache>(
-    range_frames: &[&RangeFrameBounds],
+    range_frames: &[RangeFrameBounds],
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     logical_order_value: impl ToDatumRef,
     cache_key_pk_len: usize, // this is dirty but we have no better choice
@@ -284,6 +284,14 @@ fn find_curr_for_rows_frame<'cache, const LEFT: bool>(
     } else {
         part_with_delta.upper_bound(Bound::Included(delta_key))
     };
+    let pointed_key = |cursor: CursorWithDelta<'cache, CacheKey, OwnedRow>| {
+        if LEFT {
+            cursor.peek_next().map(|(k, _)| k)
+        } else {
+            cursor.peek_prev().map(|(k, _)| k)
+        }
+    };
+
     let n_rows_to_move = if LEFT {
         frame_bounds.n_following_rows().unwrap()
     } else {
@@ -291,8 +299,7 @@ fn find_curr_for_rows_frame<'cache, const LEFT: bool>(
     };
 
     if n_rows_to_move == 0 {
-        return cursor
-            .key()
+        return pointed_key(cursor)
             .or_else(|| {
                 if LEFT {
                     part_with_delta.last_key()
@@ -304,28 +311,16 @@ fn find_curr_for_rows_frame<'cache, const LEFT: bool>(
     }
 
     for _ in 0..n_rows_to_move {
-        // Note that we have to move before check, to handle situation where the
-        // cursor is at ghost position at first.
-        if LEFT {
-            cursor.move_prev();
-        } else {
-            cursor.move_next();
-        }
-        if cursor.position().is_ghost() {
+        let res = if LEFT { cursor.prev() } else { cursor.next() };
+        if res.is_none() {
+            // we reach the end
             break;
         }
     }
-    cursor
-        .key()
-        .or_else(|| {
-            // Note the difference between this with the `n_rows_to_move == 0` case.
-            if LEFT {
-                part_with_delta.first_key()
-            } else {
-                part_with_delta.last_key()
-            }
-        })
-        .unwrap()
+
+    // We always have a valid key here, because `part_with_delta` must not be empty,
+    // and `n_rows_to_move` is always larger than 0 when we reach here.
+    pointed_key(cursor).unwrap()
 }
 
 fn find_boundary_for_rows_frame<'cache, const LEFT: bool>(
@@ -350,8 +345,18 @@ fn find_boundary_for_rows_frame<'cache, const LEFT: bool>(
     // have `curr_key` which definitely exists in the `part_with_delta`. We just find
     // the cursor pointing to it and move the cursor to frame boundary.
 
-    let mut cursor = part_with_delta.find(curr_key).unwrap();
-    assert!(!cursor.position().is_ghost());
+    let mut cursor = if LEFT {
+        part_with_delta.before(curr_key).unwrap()
+    } else {
+        part_with_delta.after(curr_key).unwrap()
+    };
+    let pointed_key = |cursor: CursorWithDelta<'cache, CacheKey, OwnedRow>| {
+        if LEFT {
+            cursor.peek_next().map(|(k, _)| k)
+        } else {
+            cursor.peek_prev().map(|(k, _)| k)
+        }
+    };
 
     let n_rows_to_move = if LEFT {
         frame_bounds.n_preceding_rows().unwrap()
@@ -360,25 +365,16 @@ fn find_boundary_for_rows_frame<'cache, const LEFT: bool>(
     };
 
     for _ in 0..n_rows_to_move {
-        if LEFT {
-            cursor.move_prev();
-        } else {
-            cursor.move_next();
-        }
-        if cursor.position().is_ghost() {
+        let res = if LEFT { cursor.prev() } else { cursor.next() };
+        if res.is_none() {
+            // we reach the end
             break;
         }
     }
-    cursor
-        .key()
-        .or_else(|| {
-            if LEFT {
-                part_with_delta.first_key()
-            } else {
-                part_with_delta.last_key()
-            }
-        })
-        .unwrap()
+
+    // We always have a valid key here, because `cursor` must point to a valid key
+    // at the beginning.
+    pointed_key(cursor).unwrap()
 }
 
 /// Given a pair of left and right state keys, calculate the leftmost (smallest) and rightmost
@@ -391,7 +387,7 @@ fn find_boundary_for_rows_frame<'cache, const LEFT: bool>(
 /// repeating. Check [`calc_logical_curr_for_range_frames`] and [`calc_logical_boundary_for_range_frames`]
 /// if you cannot understand the purpose of this function.
 fn calc_logical_ord_for_range_frames(
-    range_frames: &[&RangeFrameBounds],
+    range_frames: &[RangeFrameBounds],
     left_key: &StateKey,
     right_key: &StateKey,
     left_offset_fn: impl Fn(&RangeFrameBounds, &Datum) -> Sentinelled<Datum>,
@@ -459,7 +455,7 @@ fn calc_logical_ord_for_range_frames(
 }
 
 fn find_for_range_frames<'cache, const LEFT: bool>(
-    range_frames: &[&RangeFrameBounds],
+    range_frames: &[RangeFrameBounds],
     part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>,
     logical_order_value: impl ToDatumRef,
     cache_key_pk_len: usize,
@@ -497,11 +493,15 @@ fn find_for_range_frames<'cache, const LEFT: bool>(
             // the curr key.
             prev_key
         } else {
-            // If cursor is in ghost position, it simply means that the search key is larger
+            // If there's nothing on the left, it simply means that the search key is larger
             // than any existing key. Returning the last key in this case does no harm. Especially,
             // if the last key is largest sentinel, the caller should extend the cache rightward
             // to get possible entries with the same order value into the cache.
-            cursor.key().or_else(|| part_with_delta.last_key()).unwrap()
+            cursor
+                .peek_next()
+                .map(|(k, _)| k)
+                .or_else(|| part_with_delta.last_key())
+                .unwrap()
         }
     } else {
         let cursor = part_with_delta.upper_bound(Bound::Included(&search_key));
@@ -511,7 +511,8 @@ fn find_for_range_frames<'cache, const LEFT: bool>(
             next_key
         } else {
             cursor
-                .key()
+                .peek_prev()
+                .map(|(k, _)| k)
                 .or_else(|| part_with_delta.first_key())
                 .unwrap()
         }
@@ -1204,13 +1205,13 @@ mod tests {
             let order_type = OrderType::ascending();
 
             let range_frames = [
-                &create_range_frame(
+                create_range_frame(
                     order_data_type.clone(),
                     order_type,
                     Preceding(3i64),
                     Preceding(2i64),
                 ),
-                &create_range_frame(
+                create_range_frame(
                     order_data_type.clone(),
                     order_type,
                     Preceding(1i64),
@@ -1252,7 +1253,7 @@ mod tests {
             let order_data_type = DataType::Timestamp;
             let order_type = OrderType::descending_nulls_first();
 
-            let range_frames = [&create_range_frame(
+            let range_frames = [create_range_frame(
                 order_data_type.clone(),
                 order_type,
                 Preceding(Interval::from_month_day_usec(1, 2, 3 * 1000 * 1000)),
@@ -1320,7 +1321,7 @@ mod tests {
             expected_left: Sentinelled<ScalarImpl>,
             expected_right: Sentinelled<ScalarImpl>,
         ) {
-            let frames = if matches!(order_data_type, DataType::Int32) {
+            let range_frames = if matches!(order_data_type, DataType::Int32) {
                 [create_range_frame(
                     order_data_type.clone(),
                     order_type,
@@ -1330,7 +1331,6 @@ mod tests {
             } else {
                 panic!()
             };
-            let range_frames = frames.iter().collect::<Vec<_>>();
             let logical_order_value = Some(logical_order_value);
             let cache_key_pk_len = 1;
 
diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs
index e30bcd425a68a..16bc8065f2a00 100644
--- a/src/stream/src/executor/over_window/general.rs
+++ b/src/stream/src/executor/over_window/general.rs
@@ -17,6 +17,7 @@ use std::marker::PhantomData;
 use std::ops::RangeInclusive;
 
 use delta_btree_map::Change;
+use itertools::Itertools;
 use risingwave_common::array::stream_record::Record;
 use risingwave_common::array::Op;
 use risingwave_common::row::RowExt;
@@ -24,8 +25,11 @@ use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy;
 use risingwave_common::types::DefaultOrdered;
 use risingwave_common::util::memcmp_encoding::{self, MemcmpEncoded};
 use risingwave_common::util::sort_util::OrderType;
-use risingwave_expr::window_function::{StateKey, WindowFuncCall};
+use risingwave_expr::window_function::{
+    RangeFrameBounds, RowsFrameBounds, StateKey, WindowFuncCall,
+};
 
+use super::frame_finder::merge_rows_frames;
 use super::over_partition::{
     new_empty_partition_cache, shrink_partition_cache, CacheKey, OverPartition, PartitionCache,
     PartitionDelta,
@@ -50,8 +54,8 @@ struct ExecutorInner<S: StateStore> {
     actor_ctx: ActorContextRef,
 
     schema: Schema,
-    calls: Vec<WindowFuncCall>,
-    partition_key_indices: Vec<usize>,
+    calls: Calls,
+    deduped_part_key_indices: Vec<usize>,
     order_key_indices: Vec<usize>,
     order_key_data_types: Vec<DataType>,
     order_key_order_types: Vec<OrderType>,
@@ -88,9 +92,10 @@ impl<S: StateStore> Execute for OverWindowExecutor<S> {
 }
 
 impl<S: StateStore> ExecutorInner<S> {
+    /// Get deduplicated partition key from a full row, which happened to be the prefix of table PK.
     fn get_partition_key(&self, full_row: impl Row) -> OwnedRow {
         full_row
-            .project(&self.partition_key_indices)
+            .project(&self.deduped_part_key_indices)
             .into_owned_row()
     }
 
@@ -133,15 +138,85 @@ pub struct OverWindowExecutorArgs<S: StateStore> {
     pub cache_policy: CachePolicy,
 }
 
+/// Information about the window function calls.
+/// Contains the original calls and many other information that can be derived from the calls to avoid
+/// repeated calculation.
+pub(super) struct Calls {
+    calls: Vec<WindowFuncCall>,
+
+    /// The `ROWS` frame that is the union of all `ROWS` frames.
+    pub(super) super_rows_frame_bounds: RowsFrameBounds,
+    /// All `RANGE` frames.
+    pub(super) range_frames: Vec<RangeFrameBounds>,
+    pub(super) start_is_unbounded: bool,
+    pub(super) end_is_unbounded: bool,
+    /// Deduplicated indices of all arguments of all calls.
+    pub(super) all_arg_indices: Vec<usize>,
+
+    // TODO(rc): The following flags are used to optimize for `row_number`, `rank` and `dense_rank`.
+    // We should try our best to remove these flags while maintaining the performance in the future.
+    pub(super) numbering_only: bool,
+    pub(super) has_rank: bool,
+}
+
+impl Calls {
+    fn new(calls: Vec<WindowFuncCall>) -> Self {
+        let rows_frames = calls
+            .iter()
+            .filter_map(|call| call.frame.bounds.as_rows())
+            .collect::<Vec<_>>();
+        let super_rows_frame_bounds = merge_rows_frames(&rows_frames);
+        let range_frames = calls
+            .iter()
+            .filter_map(|call| call.frame.bounds.as_range())
+            .cloned()
+            .collect::<Vec<_>>();
+
+        let start_is_unbounded = calls
+            .iter()
+            .any(|call| call.frame.bounds.start_is_unbounded());
+        let end_is_unbounded = calls
+            .iter()
+            .any(|call| call.frame.bounds.end_is_unbounded());
+
+        let all_arg_indices = calls
+            .iter()
+            .flat_map(|call| call.args.val_indices().iter().copied())
+            .dedup()
+            .collect();
+
+        let numbering_only = calls.iter().all(|call| call.kind.is_numbering());
+        let has_rank = calls.iter().any(|call| call.kind.is_rank());
+
+        Self {
+            calls,
+            super_rows_frame_bounds,
+            range_frames,
+            start_is_unbounded,
+            end_is_unbounded,
+            all_arg_indices,
+            numbering_only,
+            has_rank,
+        }
+    }
+
+    pub(super) fn iter(&self) -> impl ExactSizeIterator<Item = &WindowFuncCall> {
+        self.calls.iter()
+    }
+
+    pub(super) fn len(&self) -> usize {
+        self.calls.len()
+    }
+}
+
 impl<S: StateStore> OverWindowExecutor<S> {
     pub fn new(args: OverWindowExecutorArgs<S>) -> Self {
+        let calls = Calls::new(args.calls);
+
         let input_info = args.input.info().clone();
         let input_schema = &input_info.schema;
 
-        let has_unbounded_frame = args
-            .calls
-            .iter()
-            .any(|call| call.frame.bounds.is_unbounded());
+        let has_unbounded_frame = calls.start_is_unbounded || calls.end_is_unbounded;
         let cache_policy = if has_unbounded_frame {
             // For unbounded frames, we finally need all entries of the partition in the cache,
             // so for simplicity we just use full cache policy for these cases.
@@ -162,13 +237,22 @@ impl<S: StateStore> OverWindowExecutor<S> {
             &input_info.pk_indices,
         );
 
+        let deduped_part_key_indices = {
+            let mut dedup = HashSet::new();
+            args.partition_key_indices
+                .iter()
+                .filter(|i| dedup.insert(**i))
+                .copied()
+                .collect()
+        };
+
         Self {
             input: args.input,
             inner: ExecutorInner {
                 actor_ctx: args.actor_ctx,
                 schema: args.schema,
-                calls: args.calls,
-                partition_key_indices: args.partition_key_indices,
+                calls,
+                deduped_part_key_indices,
                 order_key_indices: args.order_key_indices,
                 order_key_data_types,
                 order_key_order_types: args.order_key_order_types,
@@ -262,8 +346,12 @@ impl<S: StateStore> OverWindowExecutor<S> {
         chunk: StreamChunk,
         metrics: &'a OverWindowMetrics,
     ) {
-        // partition key => changes happened in the partition.
-        let mut deltas: BTreeMap<DefaultOrdered<OwnedRow>, PartitionDelta> = BTreeMap::new();
+        // (deduped) partition key => (
+        //   significant changes happened in the partition,
+        //   no-effect changes happened in the partition,
+        // )
+        let mut deltas: BTreeMap<DefaultOrdered<OwnedRow>, (PartitionDelta, PartitionDelta)> =
+            BTreeMap::new();
         // input pk of update records of which the order key is changed.
         let mut key_change_updated_pks = HashSet::new();
 
@@ -272,16 +360,16 @@ impl<S: StateStore> OverWindowExecutor<S> {
             match record {
                 Record::Insert { new_row } => {
                     let part_key = this.get_partition_key(new_row).into();
-                    let part_delta = deltas.entry(part_key).or_default();
-                    part_delta.insert(
+                    let (delta, _) = deltas.entry(part_key).or_default();
+                    delta.insert(
                         this.row_to_cache_key(new_row)?,
                         Change::Insert(new_row.into_owned_row()),
                     );
                 }
                 Record::Delete { old_row } => {
                     let part_key = this.get_partition_key(old_row).into();
-                    let part_delta = deltas.entry(part_key).or_default();
-                    part_delta.insert(this.row_to_cache_key(old_row)?, Change::Delete);
+                    let (delta, _) = deltas.entry(part_key).or_default();
+                    delta.insert(this.row_to_cache_key(old_row)?, Change::Delete);
                 }
                 Record::Update { old_row, new_row } => {
                     let old_part_key = this.get_partition_key(old_row).into();
@@ -290,23 +378,31 @@ impl<S: StateStore> OverWindowExecutor<S> {
                     let new_state_key = this.row_to_cache_key(new_row)?;
                     if old_part_key == new_part_key && old_state_key == new_state_key {
                         // not a key-change update
-                        let part_delta = deltas.entry(old_part_key).or_default();
-                        part_delta.insert(old_state_key, Change::Insert(new_row.into_owned_row()));
+                        let (delta, no_effect_delta) = deltas.entry(old_part_key).or_default();
+                        if old_row.project(&this.calls.all_arg_indices)
+                            == new_row.project(&this.calls.all_arg_indices)
+                        {
+                            // partition key, order key and arguments are all the same
+                            no_effect_delta
+                                .insert(old_state_key, Change::Insert(new_row.into_owned_row()));
+                        } else {
+                            delta.insert(old_state_key, Change::Insert(new_row.into_owned_row()));
+                        }
                     } else if old_part_key == new_part_key {
                         // order-change update, split into delete + insert, will be merged after
                         // building changes
                         key_change_updated_pks.insert(this.get_input_pk(old_row));
-                        let part_delta = deltas.entry(old_part_key).or_default();
-                        part_delta.insert(old_state_key, Change::Delete);
-                        part_delta.insert(new_state_key, Change::Insert(new_row.into_owned_row()));
+                        let (delta, _) = deltas.entry(old_part_key).or_default();
+                        delta.insert(old_state_key, Change::Delete);
+                        delta.insert(new_state_key, Change::Insert(new_row.into_owned_row()));
                     } else {
                         // partition-change update, split into delete + insert
                         // NOTE(rc): Since we append partition key to logical pk, we can't merge the
                         // delete + insert back to update later.
                         // TODO: IMO this behavior is problematic. Deep discussion is needed.
-                        let old_part_delta = deltas.entry(old_part_key).or_default();
+                        let (old_part_delta, _) = deltas.entry(old_part_key).or_default();
                         old_part_delta.insert(old_state_key, Change::Delete);
-                        let new_part_delta = deltas.entry(new_part_key).or_default();
+                        let (new_part_delta, _) = deltas.entry(new_part_key).or_default();
                         new_part_delta
                             .insert(new_state_key, Change::Insert(new_row.into_owned_row()));
                     }
@@ -320,7 +416,7 @@ impl<S: StateStore> OverWindowExecutor<S> {
         let mut chunk_builder = StreamChunkBuilder::new(this.chunk_size, this.schema.data_types());
 
         // Build final changes partition by partition.
-        for (part_key, delta) in deltas {
+        for (part_key, (delta, no_effect_delta)) in deltas {
             vars.stats.cache_lookup += 1;
             if !vars.cached_partitions.contains(&part_key.0) {
                 vars.stats.cache_miss += 1;
@@ -328,6 +424,54 @@ impl<S: StateStore> OverWindowExecutor<S> {
                     .put(part_key.0.clone(), new_empty_partition_cache());
             }
             let mut cache = vars.cached_partitions.get_mut(&part_key).unwrap();
+
+            // First, handle `Update`s that don't affect window function outputs.
+            // Be careful that changes in `delta` may (though we believe unlikely) affect the
+            // window function outputs of rows in `no_effect_delta`, so before handling `delta`
+            // we need to write all changes to state table, range cache and chunk builder.
+            for (key, change) in no_effect_delta {
+                let new_row = change.into_insert().unwrap(); // new row of an `Update`
+
+                let (old_row, from_cache) = if let Some(old_row) = cache.inner().get(&key).cloned()
+                {
+                    // Got old row from range cache.
+                    (old_row, true)
+                } else {
+                    // Retrieve old row from state table.
+                    let table_pk = (&new_row).project(this.state_table.pk_indices());
+                    // The accesses to the state table is ordered by table PK, so ideally we
+                    // can leverage the block cache under the hood.
+                    if let Some(old_row) = this.state_table.get_row(table_pk).await? {
+                        (old_row, false)
+                    } else {
+                        consistency_panic!(?part_key, ?key, ?new_row, "updating non-existing row");
+                        continue;
+                    }
+                };
+
+                // concatenate old outputs
+                let input_len = new_row.len();
+                let new_row = OwnedRow::new(
+                    new_row
+                        .into_iter()
+                        .chain(old_row.as_inner().iter().skip(input_len).cloned()) // chain old outputs
+                        .collect(),
+                );
+
+                // apply & emit the change
+                let record = Record::Update {
+                    old_row: &old_row,
+                    new_row: &new_row,
+                };
+                if let Some(chunk) = chunk_builder.append_record(record.as_ref()) {
+                    yield chunk;
+                }
+                this.state_table.write_record(record);
+                if from_cache {
+                    cache.insert(key, new_row);
+                }
+            }
+
             let mut partition = OverPartition::new(
                 &part_key,
                 &mut cache,
@@ -342,10 +486,13 @@ impl<S: StateStore> OverWindowExecutor<S> {
                 },
             );
 
+            if delta.is_empty() {
+                continue;
+            }
+
             // Build changes for current partition.
-            let (part_changes, accessed_range) = partition
-                .build_changes(&this.state_table, &this.calls, delta)
-                .await?;
+            let (part_changes, accessed_range) =
+                partition.build_changes(&this.state_table, delta).await?;
 
             for (key, record) in part_changes {
                 // Build chunk and yield if needed.
@@ -415,12 +562,15 @@ impl<S: StateStore> OverWindowExecutor<S> {
             metrics
                 .over_window_range_cache_right_miss_count
                 .inc_by(stats.right_miss_count);
+            metrics
+                .over_window_accessed_entry_count
+                .inc_by(stats.accessed_entry_count);
             metrics
                 .over_window_compute_count
                 .inc_by(stats.compute_count);
             metrics
-                .over_window_same_result_count
-                .inc_by(stats.same_result_count);
+                .over_window_same_output_count
+                .inc_by(stats.same_output_count);
 
             // Update recently accessed range for later shrinking cache.
             if !this.cache_policy.is_full()
diff --git a/src/stream/src/executor/over_window/over_partition.rs b/src/stream/src/executor/over_window/over_partition.rs
index 947c9503c3c0b..dd90105024284 100644
--- a/src/stream/src/executor/over_window/over_partition.rs
+++ b/src/stream/src/executor/over_window/over_partition.rs
@@ -19,7 +19,7 @@ use std::collections::BTreeMap;
 use std::marker::PhantomData;
 use std::ops::{Bound, RangeInclusive};
 
-use delta_btree_map::{Change, DeltaBTreeMap, PositionType};
+use delta_btree_map::{Change, DeltaBTreeMap};
 use educe::Educe;
 use futures_async_stream::for_await;
 use risingwave_common::array::stream_record::Record;
@@ -28,14 +28,12 @@ use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy;
 use risingwave_common::types::{Datum, Sentinelled};
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_common_estimate_size::collections::EstimatedBTreeMap;
-use risingwave_expr::window_function::{
-    create_window_state, RangeFrameBounds, RowsFrameBounds, StateKey, WindowFuncCall, WindowStates,
-};
+use risingwave_expr::window_function::{create_window_state, StateKey, WindowStates};
 use risingwave_storage::store::PrefetchOptions;
 use risingwave_storage::StateStore;
 use static_assertions::const_assert;
 
-use super::general::RowConverter;
+use super::general::{Calls, RowConverter};
 use crate::common::table::state_table::StateTable;
 use crate::consistency::{consistency_error, enable_strict_consistency};
 use crate::executor::over_window::frame_finder::*;
@@ -70,13 +68,13 @@ const MAGIC_CACHE_SIZE: usize = 1024;
 const MAGIC_JITTER_PREVENTION: usize = MAGIC_CACHE_SIZE / 8;
 
 pub(super) fn shrink_partition_cache(
-    this_partition_key: &OwnedRow,
+    deduped_part_key: &OwnedRow,
     range_cache: &mut PartitionCache,
     cache_policy: CachePolicy,
     recently_accessed_range: RangeInclusive<StateKey>,
 ) {
     tracing::trace!(
-        this_partition_key=?this_partition_key,
+        partition=?deduped_part_key,
         cache_policy=?cache_policy,
         recently_accessed_range=?recently_accessed_range,
         "find the range to retain in the range cache"
@@ -218,7 +216,7 @@ pub(super) fn shrink_partition_cache(
     };
 
     tracing::trace!(
-        this_partition_key=?this_partition_key,
+        partition=?deduped_part_key,
         retain_range=?(&start..=&end),
         "retain range in the range cache"
     );
@@ -247,8 +245,9 @@ pub(super) struct OverPartitionStats {
     pub right_miss_count: u64,
 
     // stats for window function state computation
+    pub accessed_entry_count: u64,
     pub compute_count: u64,
-    pub same_result_count: u64,
+    pub same_output_count: u64,
 }
 
 /// [`AffectedRange`] represents a range of keys that are affected by a delta.
@@ -289,16 +288,11 @@ impl<'a> AffectedRange<'a> {
 /// By putting this type inside `private` module, we can avoid misuse of the internal fields and
 /// methods.
 pub(super) struct OverPartition<'a, S: StateStore> {
-    this_partition_key: &'a OwnedRow,
+    deduped_part_key: &'a OwnedRow,
     range_cache: &'a mut PartitionCache,
     cache_policy: CachePolicy,
 
-    /// The `ROWS` frame that is the union of all `ROWS` frames of all window functions in this
-    /// over window executor.
-    super_rows_frame_bounds: RowsFrameBounds,
-    range_frames: Vec<&'a RangeFrameBounds>,
-    start_is_unbounded: bool,
-    end_is_unbounded: bool,
+    calls: &'a Calls,
     row_conv: RowConverter<'a>,
 
     stats: OverPartitionStats,
@@ -311,39 +305,18 @@ const MAGIC_BATCH_SIZE: usize = 512;
 impl<'a, S: StateStore> OverPartition<'a, S> {
     #[allow(clippy::too_many_arguments)]
     pub fn new(
-        this_partition_key: &'a OwnedRow,
+        deduped_part_key: &'a OwnedRow,
         cache: &'a mut PartitionCache,
         cache_policy: CachePolicy,
-        calls: &'a [WindowFuncCall],
+        calls: &'a Calls,
         row_conv: RowConverter<'a>,
     ) -> Self {
-        let rows_frames = calls
-            .iter()
-            .filter_map(|call| call.frame.bounds.as_rows())
-            .collect::<Vec<_>>();
-        // TODO(rc): maybe should avoid repeated merging
-        let super_rows_frame_bounds = merge_rows_frames(&rows_frames);
-        let range_frames = calls
-            .iter()
-            .filter_map(|call| call.frame.bounds.as_range())
-            .collect::<Vec<_>>();
-
-        let start_is_unbounded = calls
-            .iter()
-            .any(|call| call.frame.bounds.start_is_unbounded());
-        let end_is_unbounded = calls
-            .iter()
-            .any(|call| call.frame.bounds.end_is_unbounded());
-
         Self {
-            this_partition_key,
+            deduped_part_key,
             range_cache: cache,
             cache_policy,
 
-            super_rows_frame_bounds,
-            range_frames,
-            start_is_unbounded,
-            end_is_unbounded,
+            calls,
             row_conv,
 
             stats: Default::default(),
@@ -417,24 +390,32 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
     pub async fn build_changes(
         &mut self,
         table: &StateTable<S>,
-        calls: &[WindowFuncCall],
         mut delta: PartitionDelta,
     ) -> StreamExecutorResult<(
         BTreeMap<StateKey, Record<OwnedRow>>,
         Option<RangeInclusive<StateKey>>,
     )> {
+        let calls = self.calls;
         let input_schema_len = table.get_data_types().len() - calls.len();
+        let numbering_only = calls.numbering_only;
+        let has_rank = calls.has_rank;
+
+        // return values
         let mut part_changes = BTreeMap::new();
+        let mut accessed_range: Option<RangeInclusive<StateKey>> = None;
+
+        // stats
+        let mut accessed_entry_count = 0;
         let mut compute_count = 0;
-        let mut same_result_count = 0;
+        let mut same_output_count = 0;
 
-        // Find affected ranges, this also ensures that all rows in the affected ranges are loaded
-        // into the cache.
+        // Find affected ranges, this also ensures that all rows in the affected ranges are loaded into the cache.
         let (part_with_delta, affected_ranges) =
             self.find_affected_ranges(table, &mut delta).await?;
 
         let snapshot = part_with_delta.snapshot();
         let delta = part_with_delta.delta();
+        let last_delta_key = delta.last_key_value().map(|(k, _)| k.as_normal_expect());
 
         // Generate delete changes first, because deletes are skipped during iteration over
         // `part_with_delta` in the next step.
@@ -449,8 +430,6 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             }
         }
 
-        let mut accessed_range: Option<RangeInclusive<StateKey>> = None;
-
         for AffectedRange {
             first_frame_start,
             first_curr_key,
@@ -466,6 +445,8 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             assert!(last_curr_key.is_normal());
             assert!(last_frame_end.is_normal());
 
+            let last_delta_key = last_delta_key.unwrap();
+
             if let Some(accessed_range) = accessed_range.as_mut() {
                 let min_start = first_frame_start
                     .as_normal_expect()
@@ -489,12 +470,11 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             // Populate window states with the affected range of rows.
             {
                 let mut cursor = part_with_delta
-                    .find(first_frame_start)
+                    .before(first_frame_start)
                     .expect("first frame start key must exist");
-                while {
-                    let (key, row) = cursor
-                        .key_value()
-                        .expect("cursor must be valid until `last_frame_end`");
+
+                while let Some((key, row)) = cursor.next() {
+                    accessed_entry_count += 1;
 
                     for (call, state) in calls.iter().zip_eq_fast(states.iter_mut()) {
                         // TODO(rc): batch appending
@@ -507,32 +487,49 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
                                 .into(),
                         );
                     }
-                    cursor.move_next();
 
-                    key != last_frame_end
-                } {}
+                    if key == last_frame_end {
+                        break;
+                    }
+                }
             }
 
             // Slide to the first affected key. We can safely pass in `first_curr_key` here
             // because it definitely exists in the states by the definition of affected range.
             states.just_slide_to(first_curr_key.as_normal_expect())?;
-            let mut curr_key_cursor = part_with_delta.find(first_curr_key).unwrap();
+            let mut curr_key_cursor = part_with_delta.before(first_curr_key).unwrap();
             assert_eq!(
                 states.curr_key(),
-                curr_key_cursor.key().map(CacheKey::as_normal_expect)
+                curr_key_cursor
+                    .peek_next()
+                    .map(|(k, _)| k)
+                    .map(CacheKey::as_normal_expect)
             );
 
             // Slide and generate changes.
-            while {
-                let (key, row) = curr_key_cursor
-                    .key_value()
-                    .expect("cursor must be valid until `last_curr_key`");
-                let output = states.slide_no_evict_hint()?;
+            while let Some((key, row)) = curr_key_cursor.next() {
+                let mut should_stop = false;
 
+                let output = states.slide_no_evict_hint()?;
                 compute_count += 1;
+
                 let old_output = &row.as_inner()[input_schema_len..];
                 if !old_output.is_empty() && old_output == output {
-                    same_result_count += 1;
+                    same_output_count += 1;
+
+                    if numbering_only {
+                        if has_rank {
+                            // It's possible that an `Insert` doesn't affect it's ties but affects
+                            // all the following rows, so we need to check the `order_key`.
+                            if key.as_normal_expect().order_key > last_delta_key.order_key {
+                                // there won't be any more changes after this point, we can stop early
+                                should_stop = true;
+                            }
+                        } else if key.as_normal_expect() >= last_delta_key {
+                            // there won't be any more changes after this point, we can stop early
+                            should_stop = true;
+                        }
+                    }
                 }
 
                 let new_row = OwnedRow::new(
@@ -544,33 +541,28 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
                         .collect(),
                 );
 
-                match curr_key_cursor.position() {
-                    PositionType::Ghost => unreachable!(),
-                    PositionType::Snapshot | PositionType::DeltaUpdate => {
-                        // update
-                        let old_row = snapshot.get(key).unwrap().clone();
-                        if old_row != new_row {
-                            part_changes.insert(
-                                key.as_normal_expect().clone(),
-                                Record::Update { old_row, new_row },
-                            );
-                        }
-                    }
-                    PositionType::DeltaInsert => {
-                        // insert
-                        part_changes
-                            .insert(key.as_normal_expect().clone(), Record::Insert { new_row });
+                if let Some(old_row) = snapshot.get(key).cloned() {
+                    // update
+                    if old_row != new_row {
+                        part_changes.insert(
+                            key.as_normal_expect().clone(),
+                            Record::Update { old_row, new_row },
+                        );
                     }
+                } else {
+                    // insert
+                    part_changes.insert(key.as_normal_expect().clone(), Record::Insert { new_row });
                 }
 
-                curr_key_cursor.move_next();
-
-                key != last_curr_key
-            } {}
+                if should_stop || key == last_curr_key {
+                    break;
+                }
+            }
         }
 
+        self.stats.accessed_entry_count += accessed_entry_count;
         self.stats.compute_count += compute_count;
-        self.stats.same_result_count += same_result_count;
+        self.stats.same_output_count += same_output_count;
 
         Ok((part_changes, accessed_range))
     }
@@ -618,18 +610,18 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
         'a: 'delta,
         's: 'delta,
     {
-        self.ensure_delta_in_cache(table, delta).await?;
-        let delta = &*delta; // let's make it immutable
-
         if delta.is_empty() {
             return Ok((DeltaBTreeMap::new(self.range_cache.inner(), delta), vec![]));
         }
 
+        self.ensure_delta_in_cache(table, delta).await?;
+        let delta = &*delta; // let's make it immutable
+
         let delta_first = delta.first_key_value().unwrap().0.as_normal_expect();
         let delta_last = delta.last_key_value().unwrap().0.as_normal_expect();
 
         let range_frame_logical_curr =
-            calc_logical_curr_for_range_frames(&self.range_frames, delta_first, delta_last);
+            calc_logical_curr_for_range_frames(&self.calls.range_frames, delta_first, delta_last);
 
         loop {
             // TERMINATEABILITY: `extend_cache_leftward_by_n` and `extend_cache_rightward_by_n` keep
@@ -654,17 +646,17 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
 
             if need_extend_leftward {
                 self.stats.left_miss_count += 1;
-                tracing::trace!(partition=?self.this_partition_key, "partition cache left extension triggered");
+                tracing::trace!(partition=?self.deduped_part_key, "partition cache left extension triggered");
                 let left_most = self.cache_real_first_key().unwrap_or(delta_first).clone();
                 self.extend_cache_leftward_by_n(table, &left_most).await?;
             }
             if need_extend_rightward {
                 self.stats.right_miss_count += 1;
-                tracing::trace!(partition=?self.this_partition_key, "partition cache right extension triggered");
+                tracing::trace!(partition=?self.deduped_part_key, "partition cache right extension triggered");
                 let right_most = self.cache_real_last_key().unwrap_or(delta_last).clone();
                 self.extend_cache_rightward_by_n(table, &right_most).await?;
             }
-            tracing::trace!(partition=?self.this_partition_key, "partition cache extended");
+            tracing::trace!(partition=?self.deduped_part_key, "partition cache extended");
         }
     }
 
@@ -684,7 +676,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             // ensure everything is in the cache
             self.extend_cache_to_boundary(table).await?;
         } else {
-            // TODO(rc): later we should extend cache using `self.super_rows_frame_bounds` and
+            // TODO(rc): later we should extend cache using `self.calls.super_rows_frame_bounds` and
             // `range_frame_logical_curr` as hints.
 
             // ensure the cache covers all delta (if possible)
@@ -750,13 +742,13 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
         let first_key = part_with_delta.first_key().unwrap();
         let last_key = part_with_delta.last_key().unwrap();
 
-        let first_curr_key = if self.end_is_unbounded || delta_first_key == first_key {
+        let first_curr_key = if self.calls.end_is_unbounded || delta_first_key == first_key {
             // If the frame end is unbounded, or, the first key is in delta, then the frame corresponding
             // to the first key is always affected.
             first_key
         } else {
             let mut key = find_first_curr_for_rows_frame(
-                &self.super_rows_frame_bounds,
+                &self.calls.super_rows_frame_bounds,
                 part_with_delta,
                 delta_first_key,
             );
@@ -764,7 +756,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             if let Some((logical_first_curr, _)) = range_frame_logical_curr {
                 let logical_curr = logical_first_curr.as_normal_expect(); // otherwise should go `end_is_unbounded` branch
                 let new_key = find_left_for_range_frames(
-                    &self.range_frames,
+                    &self.calls.range_frames,
                     part_with_delta,
                     logical_curr,
                     cache_key_pk_len,
@@ -775,12 +767,12 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             key
         };
 
-        let last_curr_key = if self.start_is_unbounded || delta_last_key == last_key {
+        let last_curr_key = if self.calls.start_is_unbounded || delta_last_key == last_key {
             // similar to `first_curr_key`
             last_key
         } else {
             let mut key = find_last_curr_for_rows_frame(
-                &self.super_rows_frame_bounds,
+                &self.calls.super_rows_frame_bounds,
                 part_with_delta,
                 delta_last_key,
             );
@@ -788,7 +780,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             if let Some((_, logical_last_curr)) = range_frame_logical_curr {
                 let logical_curr = logical_last_curr.as_normal_expect(); // otherwise should go `start_is_unbounded` branch
                 let new_key = find_right_for_range_frames(
-                    &self.range_frames,
+                    &self.calls.range_frames,
                     part_with_delta,
                     logical_curr,
                     cache_key_pk_len,
@@ -829,18 +821,18 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
         }
 
         let range_frame_logical_boundary = calc_logical_boundary_for_range_frames(
-            &self.range_frames,
+            &self.calls.range_frames,
             first_curr_key.as_normal_expect(),
             last_curr_key.as_normal_expect(),
         );
 
-        let first_frame_start = if self.start_is_unbounded || first_curr_key == first_key {
+        let first_frame_start = if self.calls.start_is_unbounded || first_curr_key == first_key {
             // If the frame start is unbounded, or, the first curr key is the first key, then the first key
             // always need to be included in the affected range.
             first_key
         } else {
             let mut key = find_frame_start_for_rows_frame(
-                &self.super_rows_frame_bounds,
+                &self.calls.super_rows_frame_bounds,
                 part_with_delta,
                 first_curr_key,
             );
@@ -848,7 +840,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             if let Some((logical_first_start, _)) = range_frame_logical_boundary.as_ref() {
                 let logical_boundary = logical_first_start.as_normal_expect(); // otherwise should go `end_is_unbounded` branch
                 let new_key = find_left_for_range_frames(
-                    &self.range_frames,
+                    &self.calls.range_frames,
                     part_with_delta,
                     logical_boundary,
                     cache_key_pk_len,
@@ -860,12 +852,12 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
         };
         assert!(first_frame_start <= first_curr_key);
 
-        let last_frame_end = if self.end_is_unbounded || last_curr_key == last_key {
+        let last_frame_end = if self.calls.end_is_unbounded || last_curr_key == last_key {
             // similar to `first_frame_start`
             last_key
         } else {
             let mut key = find_frame_end_for_rows_frame(
-                &self.super_rows_frame_bounds,
+                &self.calls.super_rows_frame_bounds,
                 part_with_delta,
                 last_curr_key,
             );
@@ -873,7 +865,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             if let Some((_, logical_last_end)) = range_frame_logical_boundary.as_ref() {
                 let logical_boundary = logical_last_end.as_normal_expect(); // otherwise should go `end_is_unbounded` branch
                 let new_key = find_right_for_range_frames(
-                    &self.range_frames,
+                    &self.calls.range_frames,
                     part_with_delta,
                     logical_boundary,
                     cache_key_pk_len,
@@ -921,16 +913,12 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             return Ok(());
         }
 
-        tracing::trace!(partition=?self.this_partition_key, "loading the whole partition into cache");
+        tracing::trace!(partition=?self.deduped_part_key, "loading the whole partition into cache");
 
         let mut new_cache = PartitionCache::new(); // shouldn't use `new_empty_partition_cache` here because we don't want sentinels
         let sub_range: &(Bound<OwnedRow>, Bound<OwnedRow>) = &(Bound::Unbounded, Bound::Unbounded);
         let table_iter = table
-            .iter_with_prefix(
-                self.this_partition_key,
-                sub_range,
-                PrefetchOptions::default(),
-            )
+            .iter_with_prefix(self.deduped_part_key, sub_range, PrefetchOptions::default())
             .await?;
 
         #[for_await]
@@ -965,7 +953,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
         {
             // completely not overlapping, for the sake of simplicity, we re-init the cache
             tracing::debug!(
-                partition=?self.this_partition_key,
+                partition=?self.deduped_part_key,
                 cache_first=?cache_real_first_key,
                 cache_last=?cache_real_last_key,
                 range=?range,
@@ -981,7 +969,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
                 Bound::Included(self.row_conv.state_key_to_table_sub_pk(range.end())?),
             );
             tracing::debug!(
-                partition=?self.this_partition_key,
+                partition=?self.deduped_part_key,
                 table_sub_range=?table_sub_range,
                 "cache is empty, just loading the given range"
             );
@@ -1003,7 +991,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
                 ),
             );
             tracing::trace!(
-                partition=?self.this_partition_key,
+                partition=?self.deduped_part_key,
                 table_sub_range=?table_sub_range,
                 "loading the left half of given range"
             );
@@ -1022,7 +1010,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
                 Bound::Included(self.row_conv.state_key_to_table_sub_pk(range.end())?),
             );
             tracing::trace!(
-                partition=?self.this_partition_key,
+                partition=?self.deduped_part_key,
                 table_sub_range=?table_sub_range,
                 "loading the right half of given range"
             );
@@ -1135,7 +1123,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
     ) -> StreamExecutorResult<()> {
         let stream = table
             .iter_with_prefix(
-                self.this_partition_key,
+                self.deduped_part_key,
                 &table_sub_range,
                 PrefetchOptions::default(),
             )
@@ -1167,7 +1155,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             );
             let rev_stream = table
                 .rev_iter_with_prefix(
-                    self.this_partition_key,
+                    self.deduped_part_key,
                     &sub_range,
                     PrefetchOptions::default(),
                 )
@@ -1211,7 +1199,7 @@ impl<'a, S: StateStore> OverPartition<'a, S> {
             );
             let stream = table
                 .iter_with_prefix(
-                    self.this_partition_key,
+                    self.deduped_part_key,
                     &sub_range,
                     PrefetchOptions::default(),
                 )
diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs
index 6ea579afea524..40cd7cb0340ef 100644
--- a/src/stream/src/executor/project.rs
+++ b/src/stream/src/executor/project.rs
@@ -40,10 +40,8 @@ struct Inner {
     nondecreasing_expr_indices: Vec<usize>,
     /// Last seen values of nondecreasing expressions, buffered to periodically produce watermarks.
     last_nondec_expr_values: Vec<Option<ScalarImpl>>,
-
-    /// the selectivity threshold which should be in `[0,1]`. for the chunk with selectivity less
-    /// than the threshold, the Project executor will construct a new chunk before expr evaluation,
-    materialize_selectivity_threshold: f64,
+    /// Whether the stream is paused.
+    is_paused: bool,
 
     /// Whether there are likely no-op updates in the output chunks, so that eliminating them with
     /// `StreamChunk::eliminate_adjacent_noop_update` could be beneficial.
@@ -58,7 +56,6 @@ impl ProjectExecutor {
         exprs: Vec<NonStrictExpression>,
         watermark_derivations: MultiMap<usize, usize>,
         nondecreasing_expr_indices: Vec<usize>,
-        materialize_selectivity_threshold: f64,
         noop_update_hint: bool,
     ) -> Self {
         let n_nondecreasing_exprs = nondecreasing_expr_indices.len();
@@ -70,7 +67,7 @@ impl ProjectExecutor {
                 watermark_derivations,
                 nondecreasing_expr_indices,
                 last_nondec_expr_values: vec![None; n_nondecreasing_exprs],
-                materialize_selectivity_threshold,
+                is_paused: false,
                 noop_update_hint,
             },
         }
@@ -96,11 +93,6 @@ impl Inner {
         &self,
         chunk: StreamChunk,
     ) -> StreamExecutorResult<Option<StreamChunk>> {
-        let chunk = if chunk.selectivity() <= self.materialize_selectivity_threshold {
-            chunk.compact()
-        } else {
-            chunk
-        };
         let (data_chunk, ops) = chunk.into_parts();
         let mut projected_columns = Vec::new();
 
@@ -143,8 +135,13 @@ impl Inner {
 
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute(mut self, input: Executor) {
+        let mut input = input.execute();
+        let first_barrier = expect_first_barrier(&mut input).await?;
+        self.is_paused = first_barrier.is_pause_on_startup();
+        yield Message::Barrier(first_barrier);
+
         #[for_await]
-        for msg in input.execute() {
+        for msg in input {
             let msg = msg?;
             match msg {
                 Message::Watermark(w) => {
@@ -174,21 +171,36 @@ impl Inner {
                     }
                     None => continue,
                 },
-                barrier @ Message::Barrier(_) => {
-                    for (&expr_idx, value) in self
-                        .nondecreasing_expr_indices
-                        .iter()
-                        .zip_eq_fast(&mut self.last_nondec_expr_values)
-                    {
-                        if let Some(value) = std::mem::take(value) {
-                            yield Message::Watermark(Watermark::new(
-                                expr_idx,
-                                self.exprs[expr_idx].return_type(),
-                                value,
-                            ))
+                Message::Barrier(barrier) => {
+                    if !self.is_paused {
+                        for (&expr_idx, value) in self
+                            .nondecreasing_expr_indices
+                            .iter()
+                            .zip_eq_fast(&mut self.last_nondec_expr_values)
+                        {
+                            if let Some(value) = std::mem::take(value) {
+                                yield Message::Watermark(Watermark::new(
+                                    expr_idx,
+                                    self.exprs[expr_idx].return_type(),
+                                    value,
+                                ))
+                            }
                         }
                     }
-                    yield barrier;
+
+                    if let Some(mutation) = barrier.mutation.as_deref() {
+                        match mutation {
+                            Mutation::Pause => {
+                                self.is_paused = true;
+                            }
+                            Mutation::Resume => {
+                                self.is_paused = false;
+                            }
+                            _ => (),
+                        }
+                    }
+
+                    yield Message::Barrier(barrier);
                 }
             }
         }
@@ -242,7 +254,6 @@ mod tests {
             vec![test_expr],
             MultiMap::new(),
             vec![],
-            0.0,
             false,
         );
         let mut project = project.boxed().execute();
@@ -325,7 +336,6 @@ mod tests {
             vec![a_expr, b_expr, c_expr],
             MultiMap::from_iter(vec![(0, 0), (0, 1)].into_iter()),
             vec![2],
-            0.0,
             false,
         );
         let mut project = project.boxed().execute();
diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs
index 8f5c0e533bfbd..dff51a39255cf 100644
--- a/src/stream/src/executor/project_set.rs
+++ b/src/stream/src/executor/project_set.rs
@@ -93,6 +93,11 @@ impl Execute for ProjectSetExecutor {
 impl Inner {
     #[try_stream(ok = Message, error = StreamExecutorError)]
     async fn execute(self, input: Executor) {
+        let mut input = input.execute();
+        let first_barrier = expect_first_barrier(&mut input).await?;
+        let mut is_paused = first_barrier.is_pause_on_startup();
+        yield Message::Barrier(first_barrier);
+
         assert!(!self.select_list.is_empty());
         // First column will be `projected_row_id`, which represents the index in the
         // output table
@@ -104,8 +109,9 @@ impl Inner {
         let mut builder = StreamChunkBuilder::new(self.chunk_size, data_types);
 
         let mut last_nondec_expr_values = vec![None; self.nondecreasing_expr_indices.len()];
+
         #[for_await]
-        for msg in input.execute() {
+        for msg in input {
             match msg? {
                 Message::Watermark(watermark) => {
                     let watermarks = self.handle_watermark(watermark).await?;
@@ -113,21 +119,36 @@ impl Inner {
                         yield Message::Watermark(watermark)
                     }
                 }
-                m @ Message::Barrier(_) => {
-                    for (&expr_idx, value) in self
-                        .nondecreasing_expr_indices
-                        .iter()
-                        .zip_eq_fast(&mut last_nondec_expr_values)
-                    {
-                        if let Some(value) = std::mem::take(value) {
-                            yield Message::Watermark(Watermark::new(
-                                expr_idx + PROJ_ROW_ID_OFFSET,
-                                self.select_list[expr_idx].return_type(),
-                                value,
-                            ))
+                Message::Barrier(barrier) => {
+                    if !is_paused {
+                        for (&expr_idx, value) in self
+                            .nondecreasing_expr_indices
+                            .iter()
+                            .zip_eq_fast(&mut last_nondec_expr_values)
+                        {
+                            if let Some(value) = std::mem::take(value) {
+                                yield Message::Watermark(Watermark::new(
+                                    expr_idx + PROJ_ROW_ID_OFFSET,
+                                    self.select_list[expr_idx].return_type(),
+                                    value,
+                                ))
+                            }
                         }
                     }
-                    yield m
+
+                    if let Some(mutation) = barrier.mutation.as_deref() {
+                        match mutation {
+                            Mutation::Pause => {
+                                is_paused = true;
+                            }
+                            Mutation::Resume => {
+                                is_paused = false;
+                            }
+                            _ => (),
+                        }
+                    }
+
+                    yield Message::Barrier(barrier);
                 }
                 Message::Chunk(chunk) => {
                     let mut results = Vec::with_capacity(self.select_list.len());
diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs
index 058ab1142f7a9..6f85266615896 100644
--- a/src/stream/src/executor/simple_agg.rs
+++ b/src/stream/src/executor/simple_agg.rs
@@ -255,21 +255,20 @@ impl<S: StateStore> SimpleAggExecutor<S> {
         yield Message::Barrier(barrier);
 
         // This will fetch previous agg states from the intermediate state table.
-        let (agg_group, _stats) = AggGroup::create(
-            this.version,
-            None,
-            &this.agg_calls,
-            &this.agg_funcs,
-            &this.storages,
-            &this.intermediate_state_table,
-            &this.input_pk_indices,
-            this.row_count_index,
-            this.extreme_cache_size,
-            &this.input_schema,
-        )
-        .await?;
         let mut vars = ExecutionVars {
-            agg_group,
+            agg_group: AggGroup::create(
+                this.version,
+                None,
+                &this.agg_calls,
+                &this.agg_funcs,
+                &this.storages,
+                &this.intermediate_state_table,
+                &this.input_pk_indices,
+                this.row_count_index,
+                this.extreme_cache_size,
+                &this.input_schema,
+            )
+            .await?,
             distinct_dedup,
             state_changed: false,
         };
diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs
index 91f5ed876155e..206b8c4989817 100644
--- a/src/stream/src/executor/sink.rs
+++ b/src/stream/src/executor/sink.rs
@@ -295,7 +295,10 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
             match msg? {
                 Message::Watermark(w) => yield Message::Watermark(w),
                 Message::Chunk(chunk) => {
-                    assert!(!is_paused, "Should not receive any data after pause");
+                    assert!(
+                        !is_paused,
+                        "Actor {actor_id} should not receive any data after pause"
+                    );
                     log_writer.write_chunk(chunk.clone()).await?;
                     yield Message::Chunk(chunk);
                 }
@@ -505,6 +508,7 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
 
         log_reader.init().await?;
 
+        #[expect(irrefutable_let_patterns)] // false positive
         while let Err(e) = sink
             .new_log_sinker(sink_writer_param.clone())
             .and_then(|log_sinker| log_sinker.consume_log_and_sink(&mut log_reader))
@@ -517,6 +521,17 @@ impl<F: LogStoreFactory> SinkExecutor<F> {
                 actor_context.fragment_id.to_string(),
             ]);
 
+            if let Some(meta_client) = sink_writer_param.meta_client.as_ref() {
+                meta_client
+                    .add_sink_fail_evet_log(
+                        sink_writer_param.sink_id.sink_id,
+                        sink_writer_param.sink_name.clone(),
+                        sink_writer_param.connector.clone(),
+                        e.to_report_string(),
+                    )
+                    .await;
+            }
+
             match log_reader.rewind().await {
                 Ok((true, curr_vnode_bitmap)) => {
                     warn!(
diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs
index 70f0ce5f4f24b..0f3115c46c62f 100644
--- a/src/stream/src/executor/source/fs_source_executor.rs
+++ b/src/stream/src/executor/source/fs_source_executor.rs
@@ -364,8 +364,10 @@ impl<S: StateStore> FsSourceExecutor<S> {
         let barrier_stream = barrier_to_message_stream(barrier_receiver).boxed();
         let mut stream =
             StreamReaderWithPause::<true, StreamChunk>::new(barrier_stream, source_chunk_reader);
+        let mut command_paused = false;
         if start_with_paused {
             stream.pause_stream();
+            command_paused = true;
         }
 
         yield Message::Barrier(barrier);
@@ -394,7 +396,10 @@ impl<S: StateStore> FsSourceExecutor<S> {
                     Message::Barrier(barrier) => {
                         last_barrier_time = Instant::now();
                         if self_paused {
-                            stream.resume_stream();
+                            // command_paused has a higher priority.
+                            if !command_paused {
+                                stream.resume_stream();
+                            }
                             self_paused = false;
                         }
                         let epoch = barrier.epoch;
@@ -405,8 +410,14 @@ impl<S: StateStore> FsSourceExecutor<S> {
                                     self.apply_split_change(&source_desc, &mut stream, actor_splits)
                                         .await?
                                 }
-                                Mutation::Pause => stream.pause_stream(),
-                                Mutation::Resume => stream.resume_stream(),
+                                Mutation::Pause => {
+                                    command_paused = true;
+                                    stream.pause_stream()
+                                }
+                                Mutation::Resume => {
+                                    command_paused = false;
+                                    stream.resume_stream()
+                                }
                                 Mutation::Update(UpdateMutation { actor_splits, .. }) => {
                                     self.apply_split_change(
                                         &source_desc,
diff --git a/src/stream/src/executor/source/source_backfill_executor.rs b/src/stream/src/executor/source/source_backfill_executor.rs
index bd166f8082c4c..55806150bb87b 100644
--- a/src/stream/src/executor/source/source_backfill_executor.rs
+++ b/src/stream/src/executor/source/source_backfill_executor.rs
@@ -54,9 +54,21 @@ pub enum BackfillState {
     SourceCachingUp(String),
     Finished,
 }
-pub type BackfillStates = HashMap<SplitId, BackfillState>;
+pub type BackfillStates = HashMap<SplitId, BackfillStateWithProgress>;
 
-impl BackfillState {
+/// Only `state` field is the real state for fail-over.
+/// Other fields are for observability (but we still need to persist them).
+#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)]
+pub struct BackfillStateWithProgress {
+    pub state: BackfillState,
+    pub num_consumed_rows: u64,
+    /// The latest offset from upstream (inclusive). After we reach this offset, we can stop backfilling.
+    /// This is initialized with the latest available offset in the connector (if the connector provides the ability to fetch it)
+    /// so that we can finish backfilling even when upstream doesn't emit any data.
+    pub target_offset: Option<String>,
+}
+
+impl BackfillStateWithProgress {
     pub fn encode_to_json(self) -> JsonbVal {
         serde_json::to_value(self).unwrap().into()
     }
@@ -108,13 +120,13 @@ struct BackfillStage {
     ///
     /// Note: the offsets are not updated. Should use `state`'s offset to update before using it (`get_latest_unfinished_splits`).
     splits: Vec<SplitImpl>,
-    /// The latest offset from upstream (inclusive). After we reach this offset, we can stop backfilling.
-    /// This is initialized with the latest available offset in the connector (if the connector provides the ability to fetch it)
-    /// so that we can finish backfilling even when upstream doesn't emit any data.
-    target_offsets: HashMap<SplitId, Option<String>>,
 }
 
 impl BackfillStage {
+    fn total_backfilled_rows(&self) -> u64 {
+        self.states.values().map(|s| s.num_consumed_rows).sum()
+    }
+
     fn debug_assert_consistent(&self) {
         if cfg!(debug_assertions) {
             let all_splits: HashSet<_> =
@@ -123,10 +135,6 @@ impl BackfillStage {
                 self.states.keys().cloned().collect::<HashSet<_>>(),
                 all_splits
             );
-            assert_eq!(
-                self.target_offsets.keys().cloned().collect::<HashSet<_>>(),
-                all_splits
-            );
         }
     }
 
@@ -134,7 +142,7 @@ impl BackfillStage {
     fn get_latest_unfinished_splits(&self) -> StreamExecutorResult<Vec<SplitImpl>> {
         let mut unfinished_splits = Vec::new();
         for split in &self.splits {
-            let state = self.states.get(split.id().as_ref()).unwrap();
+            let state = &self.states.get(split.id().as_ref()).unwrap().state;
             match state {
                 BackfillState::Backfilling(Some(offset)) => {
                     let mut updated_split = split.clone();
@@ -152,7 +160,8 @@ impl BackfillStage {
     fn handle_upstream_row(&mut self, split_id: &str, offset: &str) -> bool {
         let mut vis = false;
         let state = self.states.get_mut(split_id).unwrap();
-        match state {
+        let state_inner = &mut state.state;
+        match state_inner {
             BackfillState::Backfilling(None) => {
                 // backfilling for this split is not started yet. Ignore this row
             }
@@ -163,12 +172,12 @@ impl BackfillStage {
                     }
                     Ordering::Equal => {
                         // backfilling for this split is finished just right.
-                        *state = BackfillState::Finished;
+                        *state_inner = BackfillState::Finished;
                     }
                     Ordering::Greater => {
                         // backfilling for this split produced more data than current source's progress.
                         // We should stop backfilling, and filter out rows from upstream with offset <= backfill_offset.
-                        *state = BackfillState::SourceCachingUp(backfill_offset.clone());
+                        *state_inner = BackfillState::SourceCachingUp(backfill_offset.clone());
                     }
                 }
             }
@@ -178,11 +187,11 @@ impl BackfillStage {
                         // Source caught up, but doesn't contain the last backfilled row.
                         // This may happen e.g., if Kafka performed compaction.
                         vis = true;
-                        *state = BackfillState::Finished;
+                        *state_inner = BackfillState::Finished;
                     }
                     Ordering::Equal => {
                         // Source just caught up with backfilling.
-                        *state = BackfillState::Finished;
+                        *state_inner = BackfillState::Finished;
                     }
                     Ordering::Greater => {
                         // Source is still behind backfilling.
@@ -194,11 +203,11 @@ impl BackfillStage {
                 // This split's backfilling is finished, we are waiting for other splits
             }
         }
-        if matches!(state, BackfillState::Backfilling(_)) {
-            *self.target_offsets.get_mut(split_id).unwrap() = Some(offset.to_string());
+        if matches!(state_inner, BackfillState::Backfilling(_)) {
+            state.target_offset = Some(offset.to_string());
         }
         if vis {
-            debug_assert_eq!(*state, BackfillState::Finished);
+            debug_assert_eq!(*state_inner, BackfillState::Finished);
         }
         vis
     }
@@ -206,10 +215,11 @@ impl BackfillStage {
     /// Updates backfill states and returns whether the row backfilled from external system is visible.
     fn handle_backfill_row(&mut self, split_id: &str, offset: &str) -> bool {
         let state = self.states.get_mut(split_id).unwrap();
-        match state {
+        state.num_consumed_rows += 1;
+        let state_inner = &mut state.state;
+        match state_inner {
             BackfillState::Backfilling(_old_offset) => {
-                let target_offset = self.target_offsets.get(split_id).unwrap();
-                if let Some(target_offset) = target_offset
+                if let Some(target_offset) = &state.target_offset
                     && compare_kafka_offset(offset, target_offset).is_ge()
                 {
                     // Note1: If target_offset = offset, it seems we can mark the state as Finished without waiting for upstream to catch up
@@ -221,9 +231,9 @@ impl BackfillStage {
                     //
                     // Note3: if target_offset is None (e.g., when upstream doesn't emit messages at all), we will
                     // keep backfilling.
-                    *state = BackfillState::SourceCachingUp(offset.to_string());
+                    *state_inner = BackfillState::SourceCachingUp(offset.to_string());
                 } else {
-                    *state = BackfillState::Backfilling(Some(offset.to_string()));
+                    *state_inner = BackfillState::Backfilling(Some(offset.to_string()));
                 }
                 true
             }
@@ -336,22 +346,20 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
         self.backfill_state_store.init_epoch(barrier.epoch);
 
         let mut backfill_states: BackfillStates = HashMap::new();
-
         for split in &owned_splits {
             let split_id = split.id();
             let backfill_state = self
                 .backfill_state_store
                 .try_recover_from_state_store(&split_id)
                 .await?
-                .unwrap_or(BackfillState::Backfilling(None));
+                .unwrap_or(BackfillStateWithProgress {
+                    state: BackfillState::Backfilling(None),
+                    num_consumed_rows: 0,
+                    target_offset: None, // init with None
+                });
             backfill_states.insert(split_id, backfill_state);
         }
         let mut backfill_stage = BackfillStage {
-            // init with None
-            target_offsets: backfill_states
-                .keys()
-                .map(|split_id| (split_id.clone(), None))
-                .collect(),
             states: backfill_states,
             splits: owned_splits,
         };
@@ -368,14 +376,14 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
             .instrument_await("source_build_reader")
             .await?;
         for (split_id, info) in &backfill_info {
+            let state = backfill_stage.states.get_mut(split_id).unwrap();
             match info {
                 BackfillInfo::NoDataToBackfill => {
-                    *backfill_stage.states.get_mut(split_id).unwrap() = BackfillState::Finished;
+                    state.state = BackfillState::Finished;
                 }
                 BackfillInfo::HasDataToBackfill { latest_offset } => {
                     // Note: later we will override it with the offset from the source message, and it's possible to become smaller than this value.
-                    *backfill_stage.target_offsets.get_mut(split_id).unwrap() =
-                        Some(latest_offset.clone());
+                    state.target_offset = Some(latest_offset.clone());
                 }
             }
         }
@@ -398,6 +406,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
 
         type PausedReader = Option<impl Stream>;
         let mut paused_reader: PausedReader = None;
+        let mut command_paused = false;
 
         macro_rules! pause_reader {
             () => {
@@ -414,6 +423,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
 
         // If the first barrier requires us to pause on startup, pause the stream.
         if barrier.is_pause_on_startup() {
+            command_paused = true;
             pause_reader!();
         }
 
@@ -495,11 +505,16 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                                 last_barrier_time = Instant::now();
 
                                 if self_paused {
-                                    backfill_stream = select_with_strategy(
-                                        input.by_ref().map(Either::Left),
-                                        paused_reader.take().expect("no paused reader to resume"),
-                                        select_strategy,
-                                    );
+                                    // command_paused has a higher priority.
+                                    if !command_paused {
+                                        backfill_stream = select_with_strategy(
+                                            input.by_ref().map(Either::Left),
+                                            paused_reader
+                                                .take()
+                                                .expect("no paused reader to resume"),
+                                            select_strategy,
+                                        );
+                                    }
                                     self_paused = false;
                                 }
 
@@ -507,16 +522,28 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                                 if let Some(ref mutation) = barrier.mutation.as_deref() {
                                     match mutation {
                                         Mutation::Pause => {
-                                            pause_reader!();
+                                            // pause_reader should not be invoked consecutively more than once.
+                                            if !command_paused {
+                                                pause_reader!();
+                                                command_paused = true;
+                                            } else {
+                                                tracing::warn!(command_paused, "unexpected pause");
+                                            }
                                         }
                                         Mutation::Resume => {
-                                            backfill_stream = select_with_strategy(
-                                                input.by_ref().map(Either::Left),
-                                                paused_reader
-                                                    .take()
-                                                    .expect("no paused reader to resume"),
-                                                select_strategy,
-                                            );
+                                            // pause_reader.take should not be invoked consecutively more than once.
+                                            if command_paused {
+                                                backfill_stream = select_with_strategy(
+                                                    input.by_ref().map(Either::Left),
+                                                    paused_reader
+                                                        .take()
+                                                        .expect("no paused reader to resume"),
+                                                    select_strategy,
+                                                );
+                                                command_paused = false;
+                                            } else {
+                                                tracing::warn!(command_paused, "unexpected resume");
+                                            }
                                         }
                                         Mutation::SourceChangeSplit(actor_splits) => {
                                             tracing::info!(
@@ -581,13 +608,11 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                                     .await?;
 
                                 if self.should_report_finished(&backfill_stage.states) {
-                                    // TODO: use a specialized progress for source
-                                    // Currently, `CreateMviewProgress` is designed for MV backfill, and rw_ddl_progress calculates
-                                    // progress based on the number of consumed rows and an estimated total number of rows from hummock.
-                                    // For now, we just rely on the same code path, and for source backfill, the progress will always be 99.99%.
                                     tracing::debug!("progress finish");
-                                    let epoch = barrier.epoch;
-                                    self.progress.finish(epoch, 114514);
+                                    self.progress.finish(
+                                        barrier.epoch,
+                                        backfill_stage.total_backfilled_rows(),
+                                    );
                                     // yield barrier after reporting progress
                                     yield Message::Barrier(barrier);
 
@@ -599,6 +624,11 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                                         break 'backfill_loop;
                                     }
                                 } else {
+                                    self.progress.update_for_source_backfill(
+                                        barrier.epoch,
+                                        backfill_stage.total_backfilled_rows(),
+                                    );
+                                    // yield barrier after reporting progress
                                     yield Message::Barrier(barrier);
                                 }
                             }
@@ -630,6 +660,13 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                         let chunk = msg?;
 
                         if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms {
+                            assert!(!command_paused, "command_paused should be false");
+                            // pause_reader should not be invoked consecutively more than once.
+                            if !self_paused {
+                                pause_reader!();
+                            } else {
+                                tracing::warn!(self_paused, "unexpected self pause");
+                            }
                             // Exceeds the max wait barrier time, the source will be paused.
                             // Currently we can guarantee the
                             // source is not paused since it received stream
@@ -640,7 +677,6 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                                 self.info.identity,
                                 last_barrier_time.elapsed()
                             );
-                            pause_reader!();
 
                             // Only update `max_wait_barrier_time_ms` to capture
                             // `barrier_interval_ms`
@@ -671,16 +707,9 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
             }
         }
 
-        let mut splits: HashSet<SplitId> = backfill_stage.states.keys().cloned().collect();
+        let mut states = backfill_stage.states;
         // Make sure `Finished` state is persisted.
-        self.backfill_state_store
-            .set_states(
-                splits
-                    .iter()
-                    .map(|s| (s.clone(), BackfillState::Finished))
-                    .collect(),
-            )
-            .await?;
+        self.backfill_state_store.set_states(states.clone()).await?;
 
         // All splits finished backfilling. Now we only forward the source data.
         #[for_await]
@@ -700,7 +729,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                                 );
                                 self.apply_split_change_forward_stage(
                                     actor_splits,
-                                    &mut splits,
+                                    &mut states,
                                     true,
                                 )
                                 .await?;
@@ -708,7 +737,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                             Mutation::Update(UpdateMutation { actor_splits, .. }) => {
                                 self.apply_split_change_forward_stage(
                                     actor_splits,
-                                    &mut splits,
+                                    &mut states,
                                     false,
                                 )
                                 .await?;
@@ -743,7 +772,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
     fn should_report_finished(&self, states: &BackfillStates) -> bool {
         states.values().all(|state| {
             matches!(
-                state,
+                state.state,
                 BackfillState::Finished | BackfillState::SourceCachingUp(_)
             )
         })
@@ -763,13 +792,13 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
     async fn backfill_finished(&self, states: &BackfillStates) -> StreamExecutorResult<bool> {
         Ok(states
             .values()
-            .all(|state| matches!(state, BackfillState::Finished))
+            .all(|state| matches!(state.state, BackfillState::Finished))
             && self
                 .backfill_state_store
                 .scan()
                 .await?
                 .into_iter()
-                .all(|state| matches!(state, BackfillState::Finished)))
+                .all(|state| matches!(state.state, BackfillState::Finished)))
     }
 
     /// For newly added splits, we do not need to backfill and can directly forward from upstream.
@@ -823,8 +852,15 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                 match backfill_state {
                     None => {
                         // Newly added split. We don't need to backfill.
-                        // Note that this branch is different from the initial barrier (BackfillState::Backfilling(None) there).
-                        target_state.insert(split_id, BackfillState::Finished);
+                        // Note that this branch is different from the initial barrier (BackfillStateInner::Backfilling(None) there).
+                        target_state.insert(
+                            split_id,
+                            BackfillStateWithProgress {
+                                state: BackfillState::Finished,
+                                num_consumed_rows: 0,
+                                target_offset: None,
+                            },
+                        );
                     }
                     Some(backfill_state) => {
                         // Migrated split. Backfill if unfinished.
@@ -858,17 +894,6 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
         }
         stage.states = target_state;
         stage.splits = target_splits;
-        let old_target_offsets = std::mem::take(&mut stage.target_offsets);
-        stage.target_offsets = stage
-            .states
-            .keys()
-            .map(|split_id| {
-                (
-                    split_id.clone(),
-                    old_target_offsets.get(split_id).cloned().flatten(),
-                )
-            })
-            .collect();
         stage.debug_assert_consistent();
         Ok(split_changed)
     }
@@ -878,12 +903,12 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
     async fn apply_split_change_forward_stage(
         &mut self,
         split_assignment: &HashMap<ActorId, Vec<SplitImpl>>,
-        splits: &mut HashSet<SplitId>,
+        states: &mut BackfillStates,
         should_trim_state: bool,
     ) -> StreamExecutorResult<()> {
         self.source_split_change_count.inc();
         if let Some(target_splits) = split_assignment.get(&self.actor_ctx.id).cloned() {
-            self.update_state_if_changed_forward_stage(target_splits, splits, should_trim_state)
+            self.update_state_if_changed_forward_stage(target_splits, states, should_trim_state)
                 .await?;
         }
 
@@ -893,7 +918,7 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
     async fn update_state_if_changed_forward_stage(
         &mut self,
         target_splits: Vec<SplitImpl>,
-        current_splits: &mut HashSet<SplitId>,
+        states: &mut BackfillStates,
         should_trim_state: bool,
     ) -> StreamExecutorResult<()> {
         let target_splits: HashSet<SplitId> = target_splits
@@ -902,23 +927,25 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
             .collect();
 
         let mut split_changed = false;
+        let mut newly_added_splits = vec![];
 
         // Checks added splits
         for split_id in &target_splits {
-            if !current_splits.contains(split_id) {
+            if !states.contains_key(split_id) {
                 split_changed = true;
 
                 let backfill_state = self
                     .backfill_state_store
                     .try_recover_from_state_store(split_id)
                     .await?;
-                match backfill_state {
+                match &backfill_state {
                     None => {
                         // Newly added split. We don't need to backfill!
+                        newly_added_splits.push(split_id.clone());
                     }
                     Some(backfill_state) => {
                         // Migrated split. It should also be finished since we are in forwarding stage.
-                        match backfill_state {
+                        match backfill_state.state {
                             BackfillState::Finished => {}
                             _ => {
                                 return Err(anyhow::anyhow!(
@@ -930,11 +957,19 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                         }
                     }
                 }
+                states.insert(
+                    split_id.clone(),
+                    backfill_state.unwrap_or(BackfillStateWithProgress {
+                        state: BackfillState::Finished,
+                        num_consumed_rows: 0,
+                        target_offset: None,
+                    }),
+                );
             }
         }
 
         // Checks dropped splits
-        for existing_split_id in current_splits.iter() {
+        for existing_split_id in states.keys() {
             if !target_splits.contains(existing_split_id) {
                 tracing::info!("split dropping detected: {}", existing_split_id);
                 split_changed = true;
@@ -947,19 +982,31 @@ impl<S: StateStore> SourceBackfillExecutorInner<S> {
                 "apply split change"
             );
 
-            let dropped_splits =
-                current_splits.extract_if(|split_id| !target_splits.contains(split_id));
+            let dropped_splits = states.extract_if(|split_id, _| !target_splits.contains(split_id));
 
             if should_trim_state {
                 // trim dropped splits' state
-                self.backfill_state_store.trim_state(dropped_splits).await?;
+                self.backfill_state_store
+                    .trim_state(dropped_splits.map(|(k, _v)| k))
+                    .await?;
             }
 
+            // For migrated splits, and existing splits, we do not need to update
+            // state store, but only for newly added splits.
             self.backfill_state_store
                 .set_states(
-                    target_splits
+                    newly_added_splits
                         .into_iter()
-                        .map(|split_id| (split_id, BackfillState::Finished))
+                        .map(|split_id| {
+                            (
+                                split_id,
+                                BackfillStateWithProgress {
+                                    state: BackfillState::Finished,
+                                    num_consumed_rows: 0,
+                                    target_offset: None,
+                                },
+                            )
+                        })
                         .collect(),
                 )
                 .await?;
diff --git a/src/stream/src/executor/source/source_backfill_state_table.rs b/src/stream/src/executor/source/source_backfill_state_table.rs
index 3579aff2ec4fb..b0ca8d363b9d7 100644
--- a/src/stream/src/executor/source/source_backfill_state_table.rs
+++ b/src/stream/src/executor/source/source_backfill_state_table.rs
@@ -23,7 +23,7 @@ use risingwave_connector::source::SplitId;
 use risingwave_pb::catalog::PbTable;
 use risingwave_storage::StateStore;
 
-use super::source_backfill_executor::{BackfillState, BackfillStates};
+use super::source_backfill_executor::{BackfillStateWithProgress, BackfillStates};
 use crate::common::table::state_table::StateTable;
 use crate::executor::error::StreamExecutorError;
 use crate::executor::StreamExecutorResult;
@@ -56,7 +56,7 @@ impl<S: StateStore> BackfillStateTableHandler<S> {
     }
 
     /// XXX: we might get stale data for other actors' writes, but it's fine?
-    pub async fn scan(&self) -> StreamExecutorResult<Vec<BackfillState>> {
+    pub async fn scan(&self) -> StreamExecutorResult<Vec<BackfillStateWithProgress>> {
         let sub_range: &(Bound<OwnedRow>, Bound<OwnedRow>) = &(Bound::Unbounded, Bound::Unbounded);
 
         let state_table_iter = self
@@ -70,7 +70,7 @@ impl<S: StateStore> BackfillStateTableHandler<S> {
             let row = item?.into_owned_row();
             let state = match row.datum_at(1) {
                 Some(ScalarRefImpl::Jsonb(jsonb_ref)) => {
-                    BackfillState::restore_from_json(jsonb_ref.to_owned_scalar())?
+                    BackfillStateWithProgress::restore_from_json(jsonb_ref.to_owned_scalar())?
                 }
                 _ => unreachable!(),
             };
@@ -80,7 +80,11 @@ impl<S: StateStore> BackfillStateTableHandler<S> {
         Ok(ret)
     }
 
-    async fn set(&mut self, key: SplitId, state: BackfillState) -> StreamExecutorResult<()> {
+    async fn set(
+        &mut self,
+        key: SplitId,
+        state: BackfillStateWithProgress,
+    ) -> StreamExecutorResult<()> {
         let row = [
             Some(Self::string_to_scalar(key.as_ref())),
             Some(ScalarImpl::Jsonb(state.encode_to_json())),
@@ -126,13 +130,13 @@ impl<S: StateStore> BackfillStateTableHandler<S> {
     pub async fn try_recover_from_state_store(
         &mut self,
         split_id: &SplitId,
-    ) -> StreamExecutorResult<Option<BackfillState>> {
+    ) -> StreamExecutorResult<Option<BackfillStateWithProgress>> {
         Ok(self
             .get(split_id)
             .await?
             .map(|row| match row.datum_at(1) {
                 Some(ScalarRefImpl::Jsonb(jsonb_ref)) => {
-                    BackfillState::restore_from_json(jsonb_ref.to_owned_scalar())
+                    BackfillStateWithProgress::restore_from_json(jsonb_ref.to_owned_scalar())
                 }
                 _ => unreachable!(),
             })
diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs
index 6d5cf710d3bb0..6e583caf739af 100644
--- a/src/stream/src/executor/source/source_executor.rs
+++ b/src/stream/src/executor/source/source_executor.rs
@@ -503,6 +503,7 @@ impl<S: StateStore> SourceExecutor<S> {
         let barrier_stream = barrier_to_message_stream(barrier_receiver).boxed();
         let mut stream =
             StreamReaderWithPause::<true, StreamChunk>::new(barrier_stream, source_chunk_reader);
+        let mut command_paused = false;
 
         // - For shared source, pause until there's a MV.
         // - If the first barrier requires us to pause on startup, pause the stream.
@@ -513,6 +514,7 @@ impl<S: StateStore> SourceExecutor<S> {
                 "source paused on startup"
             );
             stream.pause_stream();
+            command_paused = true;
         }
 
         yield Message::Barrier(barrier);
@@ -548,8 +550,11 @@ impl<S: StateStore> SourceExecutor<S> {
                     last_barrier_time = Instant::now();
 
                     if self_paused {
-                        stream.resume_stream();
                         self_paused = false;
+                        // command_paused has a higher priority.
+                        if !command_paused {
+                            stream.resume_stream();
+                        }
                     }
 
                     let epoch = barrier.epoch;
@@ -564,9 +569,14 @@ impl<S: StateStore> SourceExecutor<S> {
 
                     if let Some(mutation) = barrier.mutation.as_deref() {
                         match mutation {
-                            // XXX: Is it possible that the stream is self_paused, and we have pause mutation now? In this case, it will panic.
-                            Mutation::Pause => stream.pause_stream(),
-                            Mutation::Resume => stream.resume_stream(),
+                            Mutation::Pause => {
+                                command_paused = true;
+                                stream.pause_stream()
+                            }
+                            Mutation::Resume => {
+                                command_paused = false;
+                                stream.resume_stream()
+                            }
                             Mutation::SourceChangeSplit(actor_splits) => {
                                 tracing::info!(
                                     actor_id = self.actor_ctx.id,
diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs
index 01497c37fdab5..47a5448435b84 100644
--- a/src/stream/src/executor/watermark_filter.rs
+++ b/src/stream/src/executor/watermark_filter.rs
@@ -100,6 +100,7 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
         let first_barrier = expect_first_barrier(&mut input).await?;
         let prev_epoch = first_barrier.epoch.prev;
         table.init_epoch(first_barrier.epoch);
+        let mut is_paused = first_barrier.is_pause_on_startup();
         // The first barrier message should be propagated.
         yield Message::Barrier(first_barrier);
 
@@ -113,7 +114,9 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
 
         let mut last_checkpoint_watermark = None;
 
-        if let Some(watermark) = current_watermark.clone() {
+        if let Some(watermark) = current_watermark.clone()
+            && !is_paused
+        {
             yield Message::Watermark(Watermark::new(
                 event_time_col_idx,
                 watermark_type.clone(),
@@ -240,8 +243,20 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
                             }
                         }
                     }
-
                     table.commit(barrier.epoch).await?;
+
+                    if let Some(mutation) = barrier.mutation.as_deref() {
+                        match mutation {
+                            Mutation::Pause => {
+                                is_paused = true;
+                            }
+                            Mutation::Resume => {
+                                is_paused = false;
+                            }
+                            _ => (),
+                        }
+                    }
+
                     yield Message::Barrier(barrier);
 
                     if need_update_global_max_watermark {
@@ -253,7 +268,7 @@ impl<S: StateStore> WatermarkFilterExecutor<S> {
                         .await?;
                     }
 
-                    if is_checkpoint {
+                    if is_checkpoint && !is_paused {
                         if idle_input {
                             barrier_num_during_idle += 1;
 
diff --git a/src/stream/src/from_proto/merge.rs b/src/stream/src/from_proto/merge.rs
index d6c7ce157a931..f56090f359eef 100644
--- a/src/stream/src/from_proto/merge.rs
+++ b/src/stream/src/from_proto/merge.rs
@@ -31,6 +31,7 @@ impl MergeExecutorBuilder {
         actor_context: ActorContextRef,
         info: ExecutorInfo,
         node: &MergeNode,
+        chunk_size: usize,
     ) -> StreamResult<MergeExecutorInput> {
         let upstreams = node.get_upstream_actor_id();
         let upstream_fragment_id = node.get_upstream_fragment_id();
@@ -76,6 +77,7 @@ impl MergeExecutorBuilder {
             shared_context,
             executor_stats,
             info,
+            chunk_size,
         ))
     }
 }
@@ -98,6 +100,7 @@ impl ExecutorBuilder for MergeExecutorBuilder {
             params.actor_context,
             params.info,
             node,
+            params.env.config().developer.chunk_size,
         )?
         .into_executor(barrier_rx))
     }
diff --git a/src/stream/src/from_proto/project.rs b/src/stream/src/from_proto/project.rs
index 177045b3eba7d..62d37b4326469 100644
--- a/src/stream/src/from_proto/project.rs
+++ b/src/stream/src/from_proto/project.rs
@@ -15,7 +15,6 @@
 use multimap::MultiMap;
 use risingwave_common::util::iter_util::ZipEqFast;
 use risingwave_expr::expr::build_non_strict_from_prost;
-use risingwave_pb::expr::expr_node::RexNode;
 use risingwave_pb::stream_plan::ProjectNode;
 
 use super::*;
@@ -53,20 +52,12 @@ impl ExecutorBuilder for ProjectExecutorBuilder {
             .iter()
             .map(|idx| *idx as usize)
             .collect();
-        let extremely_light = node.get_select_list().iter().all(|expr| {
-            matches!(
-                expr.get_rex_node().unwrap(),
-                RexNode::InputRef(_) | RexNode::Constant(_)
-            )
-        });
-        let materialize_selectivity_threshold = if extremely_light { 0.0 } else { 0.5 };
         let exec = ProjectExecutor::new(
             params.actor_context,
             input,
             project_exprs,
             watermark_derivations,
             nondecreasing_expr_indices,
-            materialize_selectivity_threshold,
             node.noop_update_hint,
         );
         Ok((params.info, exec).into())
diff --git a/src/stream/src/from_proto/stream_cdc_scan.rs b/src/stream/src/from_proto/stream_cdc_scan.rs
index 3c81ecb80e859..18a47ae8a461f 100644
--- a/src/stream/src/from_proto/stream_cdc_scan.rs
+++ b/src/stream/src/from_proto/stream_cdc_scan.rs
@@ -79,9 +79,9 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder {
             .columns
             .iter()
             .filter(|col| {
-                !col.additional_column
+                col.additional_column
                     .as_ref()
-                    .is_some_and(|a_col| a_col.column_type.is_some())
+                    .is_none_or(|a_col| a_col.column_type.is_none())
             })
             .map(Into::into)
             .collect();
diff --git a/src/stream/src/from_proto/temporal_join.rs b/src/stream/src/from_proto/temporal_join.rs
index 805049a296883..99490355f326b 100644
--- a/src/stream/src/from_proto/temporal_join.rs
+++ b/src/stream/src/from_proto/temporal_join.rs
@@ -24,7 +24,9 @@ use risingwave_storage::table::batch_table::storage_table::StorageTable;
 use super::*;
 use crate::common::table::state_table::StateTable;
 use crate::executor::monitor::StreamingMetrics;
-use crate::executor::{ActorContextRef, JoinType, TemporalJoinExecutor};
+use crate::executor::{
+    ActorContextRef, JoinType, NestedLoopTemporalJoinExecutor, TemporalJoinExecutor,
+};
 use crate::task::AtomicU64Ref;
 
 pub struct TemporalJoinExecutorBuilder;
@@ -38,43 +40,6 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder {
         store: impl StateStore,
     ) -> StreamResult<Executor> {
         let table_desc: &StorageTableDesc = node.get_table_desc()?;
-        let table = {
-            let column_ids = table_desc
-                .columns
-                .iter()
-                .map(|x| ColumnId::new(x.column_id))
-                .collect_vec();
-
-            StorageTable::new_partial(
-                store.clone(),
-                column_ids,
-                params.vnode_bitmap.clone().map(Into::into),
-                table_desc,
-            )
-        };
-
-        let table_stream_key_indices = table_desc
-            .stream_key
-            .iter()
-            .map(|&k| k as usize)
-            .collect_vec();
-
-        let [source_l, source_r]: [_; 2] = params.input.try_into().unwrap();
-
-        let left_join_keys = node
-            .get_left_key()
-            .iter()
-            .map(|key| *key as usize)
-            .collect_vec();
-
-        let right_join_keys = node
-            .get_right_key()
-            .iter()
-            .map(|key| *key as usize)
-            .collect_vec();
-
-        let null_safe = node.get_null_safe().to_vec();
-
         let condition = match node.get_condition() {
             Ok(cond_prost) => Some(build_non_strict_from_prost(
                 cond_prost,
@@ -94,52 +59,118 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder {
             .iter()
             .map(|&x| x as usize)
             .collect_vec();
+        let [source_l, source_r]: [_; 2] = params.input.try_into().unwrap();
 
-        let join_key_data_types = left_join_keys
-            .iter()
-            .map(|idx| source_l.schema().fields[*idx].data_type())
-            .collect_vec();
+        if node.get_is_nested_loop() {
+            let right_table = StorageTable::new_partial(
+                store.clone(),
+                table_output_indices
+                    .iter()
+                    .map(|&x| ColumnId::new(table_desc.columns[x].column_id))
+                    .collect_vec(),
+                params.vnode_bitmap.clone().map(Into::into),
+                table_desc,
+            );
 
-        let memo_table = node.get_memo_table();
-        let memo_table = match memo_table {
-            Ok(memo_table) => {
-                let vnodes = Arc::new(
-                    params
-                        .vnode_bitmap
-                        .expect("vnodes not set for temporal join"),
-                );
-                Some(
-                    StateTable::from_table_catalog(memo_table, store.clone(), Some(vnodes.clone()))
-                        .await,
+            let dispatcher_args = NestedLoopTemporalJoinExecutorDispatcherArgs {
+                ctx: params.actor_context,
+                info: params.info.clone(),
+                left: source_l,
+                right: source_r,
+                right_table,
+                condition,
+                output_indices,
+                chunk_size: params.env.config().developer.chunk_size,
+                metrics: params.executor_stats,
+                join_type_proto: node.get_join_type()?,
+            };
+            Ok((params.info, dispatcher_args.dispatch()?).into())
+        } else {
+            let table = {
+                let column_ids = table_desc
+                    .columns
+                    .iter()
+                    .map(|x| ColumnId::new(x.column_id))
+                    .collect_vec();
+
+                StorageTable::new_partial(
+                    store.clone(),
+                    column_ids,
+                    params.vnode_bitmap.clone().map(Into::into),
+                    table_desc,
                 )
-            }
-            Err(_) => None,
-        };
-        let append_only = memo_table.is_none();
-
-        let dispatcher_args = TemporalJoinExecutorDispatcherArgs {
-            ctx: params.actor_context,
-            info: params.info.clone(),
-            left: source_l,
-            right: source_r,
-            right_table: table,
-            left_join_keys,
-            right_join_keys,
-            null_safe,
-            condition,
-            output_indices,
-            table_output_indices,
-            table_stream_key_indices,
-            watermark_epoch: params.watermark_epoch,
-            chunk_size: params.env.config().developer.chunk_size,
-            metrics: params.executor_stats,
-            join_type_proto: node.get_join_type()?,
-            join_key_data_types,
-            memo_table,
-            append_only,
-        };
+            };
+
+            let table_stream_key_indices = table_desc
+                .stream_key
+                .iter()
+                .map(|&k| k as usize)
+                .collect_vec();
+
+            let left_join_keys = node
+                .get_left_key()
+                .iter()
+                .map(|key| *key as usize)
+                .collect_vec();
+
+            let right_join_keys = node
+                .get_right_key()
+                .iter()
+                .map(|key| *key as usize)
+                .collect_vec();
+
+            let null_safe = node.get_null_safe().to_vec();
+
+            let join_key_data_types = left_join_keys
+                .iter()
+                .map(|idx| source_l.schema().fields[*idx].data_type())
+                .collect_vec();
 
-        Ok((params.info, dispatcher_args.dispatch()?).into())
+            let memo_table = node.get_memo_table();
+            let memo_table = match memo_table {
+                Ok(memo_table) => {
+                    let vnodes = Arc::new(
+                        params
+                            .vnode_bitmap
+                            .expect("vnodes not set for temporal join"),
+                    );
+                    Some(
+                        StateTable::from_table_catalog(
+                            memo_table,
+                            store.clone(),
+                            Some(vnodes.clone()),
+                        )
+                        .await,
+                    )
+                }
+                Err(_) => None,
+            };
+            let append_only = memo_table.is_none();
+
+            let dispatcher_args = TemporalJoinExecutorDispatcherArgs {
+                ctx: params.actor_context,
+                info: params.info.clone(),
+                left: source_l,
+                right: source_r,
+                right_table: table,
+                left_join_keys,
+                right_join_keys,
+                null_safe,
+                condition,
+                output_indices,
+                table_output_indices,
+                table_stream_key_indices,
+                watermark_epoch: params.watermark_epoch,
+                chunk_size: params.env.config().developer.chunk_size,
+                metrics: params.executor_stats,
+                join_type_proto: node.get_join_type()?,
+                join_key_data_types,
+                memo_table,
+                append_only,
+            };
+
+            Ok((params.info, dispatcher_args.dispatch()?).into())
+        }
     }
 }
 
@@ -221,3 +252,49 @@ impl<S: StateStore> HashKeyDispatcher for TemporalJoinExecutorDispatcherArgs<S>
         &self.join_key_data_types
     }
 }
+
+struct NestedLoopTemporalJoinExecutorDispatcherArgs<S: StateStore> {
+    ctx: ActorContextRef,
+    info: ExecutorInfo,
+    left: Executor,
+    right: Executor,
+    right_table: StorageTable<S>,
+    condition: Option<NonStrictExpression>,
+    output_indices: Vec<usize>,
+    chunk_size: usize,
+    metrics: Arc<StreamingMetrics>,
+    join_type_proto: JoinTypeProto,
+}
+
+impl<S: StateStore> NestedLoopTemporalJoinExecutorDispatcherArgs<S> {
+    fn dispatch(self) -> StreamResult<Box<dyn Execute>> {
+        /// This macro helps to fill the const generic type parameter.
+        macro_rules! build {
+            ($join_type:ident) => {
+                Ok(Box::new(NestedLoopTemporalJoinExecutor::<
+                    S,
+                    { JoinType::$join_type },
+                >::new(
+                    self.ctx,
+                    self.info,
+                    self.left,
+                    self.right,
+                    self.right_table,
+                    self.condition,
+                    self.output_indices,
+                    self.metrics,
+                    self.chunk_size,
+                )))
+            };
+        }
+        match self.join_type_proto {
+            JoinTypeProto::Inner => {
+                build!(Inner)
+            }
+            JoinTypeProto::LeftOuter => {
+                build!(LeftOuter)
+            }
+            _ => unreachable!(),
+        }
+    }
+}
diff --git a/src/stream/src/lib.rs b/src/stream/src/lib.rs
index 577b829945620..06893b16c2455 100644
--- a/src/stream/src/lib.rs
+++ b/src/stream/src/lib.rs
@@ -34,7 +34,6 @@
 #![feature(exact_size_is_empty)]
 #![feature(impl_trait_in_assoc_type)]
 #![feature(test)]
-#![feature(is_sorted)]
 #![feature(btree_cursors)]
 #![feature(assert_matches)]
 #![feature(try_blocks)]
diff --git a/src/stream/src/task/barrier_manager/progress.rs b/src/stream/src/task/barrier_manager/progress.rs
index 9b2820bb3bfed..dba8f5050627a 100644
--- a/src/stream/src/task/barrier_manager/progress.rs
+++ b/src/stream/src/task/barrier_manager/progress.rs
@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use std::assert_matches::assert_matches;
 use std::fmt::{Display, Formatter};
 
 use risingwave_common::util::epoch::EpochPair;
@@ -27,23 +28,32 @@ type ConsumedRows = u64;
 
 #[derive(Debug, Clone, Copy)]
 pub(crate) enum BackfillState {
-    ConsumingUpstream(ConsumedEpoch, ConsumedRows),
-    Done(ConsumedRows),
+    ConsumingUpstreamTableOrSource(ConsumedEpoch, ConsumedRows),
+    DoneConsumingUpstreamTableOrSource(ConsumedRows),
+    ConsumingLogStore { pending_barrier_num: usize },
+    DoneConsumingLogStore,
 }
 
 impl BackfillState {
     pub fn to_pb(self, actor_id: ActorId) -> PbCreateMviewProgress {
+        let (done, consumed_epoch, consumed_rows, pending_barrier_num) = match self {
+            BackfillState::ConsumingUpstreamTableOrSource(consumed_epoch, consumed_rows) => {
+                (false, consumed_epoch, consumed_rows, 0)
+            }
+            BackfillState::DoneConsumingUpstreamTableOrSource(consumed_rows) => {
+                (true, 0, consumed_rows, 0)
+            } // unused field for done
+            BackfillState::ConsumingLogStore {
+                pending_barrier_num,
+            } => (false, 0, 0, pending_barrier_num as _),
+            BackfillState::DoneConsumingLogStore => (true, 0, 0, 0),
+        };
         PbCreateMviewProgress {
             backfill_actor_id: actor_id,
-            done: matches!(self, BackfillState::Done(_)),
-            consumed_epoch: match self {
-                BackfillState::ConsumingUpstream(consumed_epoch, _) => consumed_epoch,
-                BackfillState::Done(_) => 0, // unused field for done
-            },
-            consumed_rows: match self {
-                BackfillState::ConsumingUpstream(_, consumed_rows) => consumed_rows,
-                BackfillState::Done(consumed_rows) => consumed_rows,
-            },
+            done,
+            consumed_epoch,
+            consumed_rows,
+            pending_barrier_num,
         }
     }
 }
@@ -51,10 +61,27 @@ impl BackfillState {
 impl Display for BackfillState {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            BackfillState::ConsumingUpstream(epoch, rows) => {
-                write!(f, "ConsumingUpstream(epoch: {}, rows: {})", epoch, rows)
+            BackfillState::ConsumingUpstreamTableOrSource(epoch, rows) => {
+                write!(
+                    f,
+                    "ConsumingUpstreamTable(epoch: {}, rows: {})",
+                    epoch, rows
+                )
+            }
+            BackfillState::DoneConsumingUpstreamTableOrSource(rows) => {
+                write!(f, "DoneConsumingUpstreamTable(rows: {})", rows)
+            }
+            BackfillState::ConsumingLogStore {
+                pending_barrier_num,
+            } => {
+                write!(
+                    f,
+                    "ConsumingLogStore(pending_barrier_num: {pending_barrier_num})"
+                )
+            }
+            BackfillState::DoneConsumingLogStore => {
+                write!(f, "DoneConsumingLogStore")
             }
-            BackfillState::Done(rows) => write!(f, "Done(rows: {})", rows),
         }
     }
 }
@@ -165,7 +192,7 @@ impl CreateMviewProgressReporter {
         current_consumed_rows: ConsumedRows,
     ) {
         match self.state {
-            Some(BackfillState::ConsumingUpstream(last, last_consumed_rows)) => {
+            Some(BackfillState::ConsumingUpstreamTableOrSource(last, last_consumed_rows)) => {
                 assert!(
                     last < consumed_epoch,
                     "last_epoch: {:#?} must be greater than consumed epoch: {:#?}",
@@ -174,22 +201,90 @@ impl CreateMviewProgressReporter {
                 );
                 assert!(last_consumed_rows <= current_consumed_rows);
             }
-            Some(BackfillState::Done(_)) => unreachable!(),
+            Some(state) => {
+                panic!(
+                    "should not update consuming progress at invalid state: {:?}",
+                    state
+                )
+            }
+            None => {}
+        };
+        self.update_inner(
+            epoch,
+            BackfillState::ConsumingUpstreamTableOrSource(consumed_epoch, current_consumed_rows),
+        );
+    }
+
+    /// The difference from [`Self::update`] (MV backfill) is that we
+    /// don't care `ConsumedEpoch` here.
+    pub fn update_for_source_backfill(
+        &mut self,
+        epoch: EpochPair,
+        current_consumed_rows: ConsumedRows,
+    ) {
+        match self.state {
+            Some(BackfillState::ConsumingUpstreamTableOrSource(
+                dummy_last_epoch,
+                _last_consumed_rows,
+            )) => {
+                debug_assert_eq!(dummy_last_epoch, 0);
+            }
+            Some(state) => {
+                panic!(
+                    "should not update consuming progress at invalid state: {:?}",
+                    state
+                )
+            }
             None => {}
         };
         self.update_inner(
             epoch,
-            BackfillState::ConsumingUpstream(consumed_epoch, current_consumed_rows),
+            // fill a dummy ConsumedEpoch
+            BackfillState::ConsumingUpstreamTableOrSource(0, current_consumed_rows),
         );
     }
 
     /// Finish the progress. If the progress is already finished, then perform no-op.
     /// `current_epoch` should be provided to locate the barrier under concurrent checkpoint.
     pub fn finish(&mut self, epoch: EpochPair, current_consumed_rows: ConsumedRows) {
-        if let Some(BackfillState::Done(_)) = self.state {
+        if let Some(BackfillState::DoneConsumingUpstreamTableOrSource(_)) = self.state {
             return;
         }
-        self.update_inner(epoch, BackfillState::Done(current_consumed_rows));
+        self.update_inner(
+            epoch,
+            BackfillState::DoneConsumingUpstreamTableOrSource(current_consumed_rows),
+        );
+    }
+
+    pub(crate) fn update_create_mview_log_store_progress(
+        &mut self,
+        epoch: EpochPair,
+        pending_barrier_num: usize,
+    ) {
+        assert_matches!(
+            self.state,
+            Some(BackfillState::DoneConsumingUpstreamTableOrSource(_))
+                | Some(BackfillState::ConsumingLogStore { .. }),
+            "cannot update log store progress at state {:?}",
+            self.state
+        );
+        self.update_inner(
+            epoch,
+            BackfillState::ConsumingLogStore {
+                pending_barrier_num,
+            },
+        );
+    }
+
+    pub(crate) fn finish_consuming_log_store(&mut self, epoch: EpochPair) {
+        assert_matches!(
+            self.state,
+            Some(BackfillState::DoneConsumingUpstreamTableOrSource(_))
+                | Some(BackfillState::ConsumingLogStore { .. }),
+            "cannot finish log store progress at state {:?}",
+            self.state
+        );
+        self.update_inner(epoch, BackfillState::DoneConsumingLogStore);
     }
 }
 
@@ -201,7 +296,7 @@ impl LocalBarrierManager {
     ///
     /// When all backfill executors of the creating mview finish, the creation progress will be done at
     /// frontend and the mview will be exposed to the user.
-    pub fn register_create_mview_progress(
+    pub(crate) fn register_create_mview_progress(
         &self,
         backfill_actor_id: ActorId,
     ) -> CreateMviewProgressReporter {
diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs
index 83a7137d4403b..799f98fc1a848 100644
--- a/src/stream/src/task/stream_manager.rs
+++ b/src/stream/src/task/stream_manager.rs
@@ -321,6 +321,7 @@ impl StreamActorManager {
         upstream_node: &StreamNode,
         actor_context: &ActorContextRef,
         shared_context: &Arc<SharedContext>,
+        chunk_size: usize,
     ) -> StreamResult<MergeExecutorInput> {
         let info = Self::get_executor_info(
             upstream_node,
@@ -337,6 +338,7 @@ impl StreamActorManager {
             actor_context.clone(),
             info,
             upstream_merge,
+            chunk_size,
         )
     }
 
@@ -352,8 +354,13 @@ impl StreamActorManager {
         state_store: impl StateStore,
     ) -> StreamResult<Executor> {
         let [upstream_node, _]: &[_; 2] = stream_node.input.as_slice().try_into().unwrap();
-        let upstream =
-            self.create_snapshot_backfill_input(upstream_node, actor_context, shared_context)?;
+        let chunk_size = env.config().developer.chunk_size;
+        let upstream = self.create_snapshot_backfill_input(
+            upstream_node,
+            actor_context,
+            shared_context,
+            chunk_size,
+        )?;
 
         let table_desc: &StorageTableDesc = node.get_table_desc()?;
 
@@ -387,7 +394,7 @@ impl StreamActorManager {
             output_indices,
             actor_context.clone(),
             progress,
-            env.config().developer.chunk_size,
+            chunk_size,
             node.rate_limit.map(|x| x as _),
             barrier_rx,
             self.streaming_metrics.clone(),
@@ -404,10 +411,7 @@ impl StreamActorManager {
             troubled_info.identity = format!("{} (troubled)", info.identity);
             Ok((
                 info,
-                TroublemakerExecutor::new(
-                    (troubled_info, executor).into(),
-                    env.config().developer.chunk_size,
-                ),
+                TroublemakerExecutor::new((troubled_info, executor).into(), chunk_size),
             )
                 .into())
         } else {
diff --git a/src/stream/tests/integration_tests/project_set.rs b/src/stream/tests/integration_tests/project_set.rs
index 543f710b61b75..ce0bef7a832de 100644
--- a/src/stream/tests/integration_tests/project_set.rs
+++ b/src/stream/tests/integration_tests/project_set.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use multimap::MultiMap;
+use risingwave_common::util::epoch::test_epoch;
 use risingwave_expr::table_function::repeat;
 use risingwave_stream::executor::ProjectSetExecutor;
 use risingwave_stream::task::ActorEvalErrorReport;
@@ -60,6 +61,7 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) {
 async fn test_project_set() {
     let (mut tx, mut project_set) = create_executor();
 
+    tx.push_barrier(test_epoch(1), false);
     tx.push_chunk(StreamChunk::from_pretty(
         " I I
         + 1 4
@@ -76,6 +78,7 @@ async fn test_project_set() {
     check_until_pending(
         &mut project_set,
         expect_test::expect![[r#"
+            - !barrier 1
             - !chunk |-
               +---+---+---+---+---+---+
               | + | 0 | 5 | 2 | 1 | 2 |
diff --git a/src/tests/regress/data/expected/create_table.out b/src/tests/regress/data/expected/create_table.out
index a958b84979992..6248ca2c21222 100644
--- a/src/tests/regress/data/expected/create_table.out
+++ b/src/tests/regress/data/expected/create_table.out
@@ -1016,7 +1016,7 @@ create table parted_notnull_inh_test1 partition of parted_notnull_inh_test (a no
 insert into parted_notnull_inh_test (b) values (null);
 ERROR:  null value in column "b" of relation "parted_notnull_inh_test1" violates not-null constraint
 DETAIL:  Failing row contains (1, null).
--- note that while b's default is overriden, a's default is preserved
+-- note that while b's default is overridden, a's default is preserved
 \d parted_notnull_inh_test1
       Table "public.parted_notnull_inh_test1"
  Column |  Type   | Collation | Nullable | Default 
diff --git a/src/tests/regress/data/sql/create_table.sql b/src/tests/regress/data/sql/create_table.sql
index cc41f58ba22a7..7f549e966a8f0 100644
--- a/src/tests/regress/data/sql/create_table.sql
+++ b/src/tests/regress/data/sql/create_table.sql
@@ -801,7 +801,7 @@ CREATE TABLE part_c_1_10 PARTITION OF part_c FOR VALUES FROM (1) TO (10);
 create table parted_notnull_inh_test (a int default 1, b int not null default 0) partition by list (a);
 create table parted_notnull_inh_test1 partition of parted_notnull_inh_test (a not null, b default 1) for values in (1);
 insert into parted_notnull_inh_test (b) values (null);
--- note that while b's default is overriden, a's default is preserved
+-- note that while b's default is overridden, a's default is preserved
 \d parted_notnull_inh_test1
 drop table parted_notnull_inh_test;
 
diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml
index 190a8396001b5..f5b3fc6145e16 100644
--- a/src/tests/simulation/Cargo.toml
+++ b/src/tests/simulation/Cargo.toml
@@ -23,7 +23,7 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] }
 glob = "0.3"
 itertools = { workspace = true }
 lru = { workspace = true }
-madsim = "0.2.30"
+madsim = "0.2.31"
 maplit = "1"
 paste = "1"
 pin-project = "1.1"
diff --git a/src/tests/simulation/src/client.rs b/src/tests/simulation/src/client.rs
index 84717583eb422..65ec200b05c37 100644
--- a/src/tests/simulation/src/client.rs
+++ b/src/tests/simulation/src/client.rs
@@ -53,7 +53,7 @@ where
     stmts_iter: core::iter::Rev<Iter<'b, String, String>>,
 }
 
-impl<'a, 'b> SetStmtsIterator<'a, 'b> {
+impl<'a> SetStmtsIterator<'a, '_> {
     fn new(stmts: &'a SetStmts) -> Self {
         Self {
             _stmts: stmts,
diff --git a/src/tests/simulation/src/cluster.rs b/src/tests/simulation/src/cluster.rs
index 3b41c82afab06..d7296c5eea826 100644
--- a/src/tests/simulation/src/cluster.rs
+++ b/src/tests/simulation/src/cluster.rs
@@ -37,6 +37,7 @@ use risingwave_common::util::tokio_util::sync::CancellationToken;
 use risingwave_object_store::object::sim::SimServer as ObjectStoreSimServer;
 use risingwave_pb::common::WorkerNode;
 use sqllogictest::AsyncDB;
+use tempfile::NamedTempFile;
 #[cfg(not(madsim))]
 use tokio::runtime::Handle;
 use uuid::Uuid;
@@ -88,9 +89,6 @@ pub struct Configuration {
 
     /// Queries to run per session.
     pub per_session_queries: Arc<Vec<String>>,
-
-    /// dir to store SQL backend sqlite db
-    pub sqlite_data_dir: Option<PathBuf>,
 }
 
 impl Default for Configuration {
@@ -118,7 +116,6 @@ metrics_level = "Disabled"
             compactor_nodes: 1,
             compute_node_cores: 1,
             per_session_queries: vec![].into(),
-            sqlite_data_dir: None,
         }
     }
 }
@@ -158,7 +155,7 @@ impl Configuration {
 
     pub fn for_scale_shared_source() -> Self {
         let mut conf = Self::for_scale();
-        conf.per_session_queries = vec!["SET ENABLE_SHARED_SOURCE = true;".into()].into();
+        conf.per_session_queries = vec!["SET STREAMING_USE_SHARED_SOURCE = true;".into()].into();
         conf
     }
 
@@ -206,7 +203,6 @@ metrics_level = "Disabled"
                 "create view if not exists mview_parallelism as select m.name, tf.parallelism from rw_materialized_views m, rw_table_fragments tf where m.id = tf.table_id;".into(),
             ]
                 .into(),
-            ..Default::default()
         }
     }
 
@@ -253,7 +249,6 @@ metrics_level = "Disabled"
             compute_node_cores: 1,
             per_session_queries: vec!["SET STREAMING_USE_ARRANGEMENT_BACKFILL = true;".into()]
                 .into(),
-            ..Default::default()
         }
     }
 
@@ -300,7 +295,6 @@ metrics_level = "Disabled"
             compactor_nodes: 1,
             compute_node_cores: 1,
             per_session_queries: vec![].into(),
-            ..Default::default()
         }
     }
 }
@@ -327,6 +321,8 @@ pub struct Cluster {
     pub(crate) client: NodeHandle,
     #[cfg(madsim)]
     pub(crate) ctl: NodeHandle,
+    #[cfg(madsim)]
+    pub(crate) sqlite_file_handle: NamedTempFile,
 }
 
 impl Cluster {
@@ -399,39 +395,12 @@ impl Cluster {
         }
         std::env::set_var("RW_META_ADDR", meta_addrs.join(","));
 
-        // FIXME: some tests like integration tests will run concurrently,
-        // resulting in connecting to the same sqlite file if they're using the same seed.
-        let file_path = if let Some(sqlite_data_dir) = conf.sqlite_data_dir.as_ref() {
-            format!(
-                "{}/stest-{}-{}.sqlite",
-                sqlite_data_dir.display(),
-                handle.seed(),
-                Uuid::new_v4()
-            )
-        } else {
-            format!("./stest-{}-{}.sqlite", handle.seed(), Uuid::new_v4())
-        };
-        if std::fs::exists(&file_path).unwrap() {
-            panic!(
-                "sqlite file already exists and used by other cluster: {}",
-                file_path
-            )
-        }
+        let sqlite_file_handle: NamedTempFile = NamedTempFile::new().unwrap();
+        let file_path = sqlite_file_handle.path().display().to_string();
+        tracing::info!(?file_path, "sqlite_file_path");
         let sql_endpoint = format!("sqlite://{}?mode=rwc", file_path);
         let backend_args = vec!["--backend", "sql", "--sql-endpoint", &sql_endpoint];
 
-        // FIXME(kwannoel):
-        // Currently we just use the on-disk version,
-        // but it can lead to randomness due to disk io.
-        // We can use shared in-memory db instead.
-        // However sqlite cannot be started inside meta.
-        // Because if cluster stops, then this db will be dropped.
-        // We must instantiate it outside, not just pass the path in.
-        // let sqlite_path = format!(
-        //     "sqlite::file:memdb{}?mode=memory&cache=shared",
-        //     Uuid::new_v4()
-        // );
-
         // meta node
         for i in 1..=conf.meta_nodes {
             let args = [
@@ -570,6 +539,7 @@ impl Cluster {
             handle,
             client,
             ctl,
+            sqlite_file_handle,
         })
     }
 
@@ -888,33 +858,6 @@ impl Cluster {
     }
 }
 
-#[cfg_or_panic(madsim)]
-impl Drop for Cluster {
-    fn drop(&mut self) {
-        // FIXME: remove it when deprecate the on-disk version.
-        let default_path = PathBuf::from(".");
-        let sqlite_data_dir = self
-            .config
-            .sqlite_data_dir
-            .as_ref()
-            .unwrap_or_else(|| &default_path);
-        for entry in std::fs::read_dir(sqlite_data_dir).unwrap() {
-            let entry = entry.unwrap();
-            let path = entry.path();
-            if path
-                .file_name()
-                .unwrap()
-                .to_str()
-                .unwrap()
-                .starts_with(&format!("stest-{}-", self.handle.seed()))
-            {
-                std::fs::remove_file(path).unwrap();
-                break;
-            }
-        }
-    }
-}
-
 type SessionRequest = (
     String,                          // query sql
     oneshot::Sender<Result<String>>, // channel to send result back
diff --git a/src/tests/simulation/src/main.rs b/src/tests/simulation/src/main.rs
index 6ef89afc5eb93..4d3122c486d94 100644
--- a/src/tests/simulation/src/main.rs
+++ b/src/tests/simulation/src/main.rs
@@ -14,8 +14,6 @@
 
 #![cfg_attr(not(madsim), allow(dead_code))]
 
-use std::path::PathBuf;
-
 use cfg_or_panic::cfg_or_panic;
 use clap::Parser;
 
@@ -124,10 +122,6 @@ pub struct Args {
     #[clap(long)]
     run_differential_tests: bool,
 
-    /// dir to store sqlite backend data of meta node
-    #[clap(long)]
-    sqlite_data_dir: Option<PathBuf>,
-
     #[arg(short, long)]
     e2e_extended_test: bool,
 
@@ -166,7 +160,6 @@ async fn main() {
         compactor_nodes: args.compactor_nodes,
         compute_node_cores: args.compute_node_cores,
         meta_nodes: args.meta_nodes,
-        sqlite_data_dir: args.sqlite_data_dir,
         per_session_queries: if args.use_arrangement_backfill {
             vec!["SET STREAMING_USE_ARRANGEMENT_BACKFILL = true;".to_string()].into()
         } else {
diff --git a/src/tests/simulation/tests/integration_tests/README.md b/src/tests/simulation/tests/integration_tests/README.md
index 974200bb57aa6..9e0f46307acae 100644
--- a/src/tests/simulation/tests/integration_tests/README.md
+++ b/src/tests/simulation/tests/integration_tests/README.md
@@ -1,4 +1,4 @@
 # Simulation Integration Tests
 
-Take a look at [Developer Guide](../../../../../docs/developer-guide.md) for more information
-about running the integration tests locally.
\ No newline at end of file
+Take a look at [Developer Guide](https://risingwavelabs.github.io/risingwave/tests/intro.html#integration-tests) for more information
+about running the integration tests locally.
diff --git a/src/tests/simulation/tests/integration_tests/backfill_tests.rs b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
index fb7cb3db6fb8b..7d6569f6bae40 100644
--- a/src/tests/simulation/tests/integration_tests/backfill_tests.rs
+++ b/src/tests/simulation/tests/integration_tests/backfill_tests.rs
@@ -262,7 +262,7 @@ async fn test_arrangement_backfill_progress() -> Result<()> {
     let progress = session
         .run("SELECT progress FROM rw_catalog.rw_ddl_progress")
         .await?;
-    let progress = progress.replace('%', "");
+    let progress = progress.split_once("%").unwrap().0;
     let progress = progress.parse::<f64>().unwrap();
     assert!(
         (0.5..1.5).contains(&progress),
@@ -276,7 +276,7 @@ async fn test_arrangement_backfill_progress() -> Result<()> {
     let progress = session
         .run("SELECT progress FROM rw_catalog.rw_ddl_progress")
         .await?;
-    let progress = progress.replace('%', "");
+    let progress = progress.split_once("%").unwrap().0;
     let progress = progress.parse::<f64>().unwrap();
     assert!(
         (prev_progress - 0.5..prev_progress + 1.5).contains(&progress),
diff --git a/src/tests/simulation/tests/integration_tests/compaction/mod.rs b/src/tests/simulation/tests/integration_tests/compaction/mod.rs
index f9f23ac72387c..8bbd0cbd31389 100644
--- a/src/tests/simulation/tests/integration_tests/compaction/mod.rs
+++ b/src/tests/simulation/tests/integration_tests/compaction/mod.rs
@@ -122,18 +122,25 @@ async fn test_vnode_watermark_reclaim_impl(
         .parse::<u64>()
         .unwrap();
 
-    // Move the table to a dedicated group to prevent its vnode watermark from being reclaimed during the compaction of other tables.
-    cluster.split_compaction_group(2, table_id).await.unwrap();
     tokio::time::sleep(Duration::from_secs(5)).await;
-    let compaction_group_id = session
-        .run(format!(
-            "SELECT id FROM rw_hummock_compaction_group_configs where member_tables @> '[{}]'::jsonb;",
-            table_id
-        ))
+    async fn compaction_group_id_by_table_id(session: &mut Session, table_id: u64) -> u64 {
+        session
+            .run(format!(
+                "SELECT id FROM rw_hummock_compaction_group_configs where member_tables @> '[{}]'::jsonb;",
+                table_id
+            ))
+            .await
+            .unwrap()
+            .parse::<u64>()
+            .unwrap()
+    }
+    let original_compaction_group_id = compaction_group_id_by_table_id(session, table_id).await;
+    // Move the table to a dedicated group to prevent its vnode watermark from being reclaimed during the compaction of other tables.
+    cluster
+        .split_compaction_group(original_compaction_group_id, table_id)
         .await
-        .unwrap()
-        .parse::<u64>()
         .unwrap();
+    let compaction_group_id = compaction_group_id_by_table_id(session, table_id).await;
 
     session
         .run("INSERT INTO t2 VALUES (now(), 1);")
diff --git a/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs
index 9cc9a6a9ab716..1377fe8730f27 100644
--- a/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs
+++ b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs
@@ -318,7 +318,7 @@ async fn test_high_barrier_latency_cancel(config: Configuration) -> Result<()> {
             .await
             .unwrap();
         tracing::info!(progress, "get progress before cancel stream job");
-        let progress = progress.replace('%', "");
+        let progress = progress.split_once("%").unwrap().0;
         let progress = progress.parse::<f64>().unwrap();
         if progress >= 0.01 {
             break;
diff --git a/src/tests/simulation/tests/integration_tests/sink/err_isolation.rs b/src/tests/simulation/tests/integration_tests/sink/err_isolation.rs
index bb153f0799961..124653946b872 100644
--- a/src/tests/simulation/tests/integration_tests/sink/err_isolation.rs
+++ b/src/tests/simulation/tests/integration_tests/sink/err_isolation.rs
@@ -64,3 +64,42 @@ async fn test_sink_decouple_err_isolation() -> Result<()> {
 
     Ok(())
 }
+
+#[tokio::test]
+async fn test_sink_error_event_logs() -> Result<()> {
+    let mut cluster = start_sink_test_cluster().await?;
+
+    let source_parallelism = 6;
+
+    let test_sink = SimulationTestSink::register_new();
+    test_sink.set_err_rate(1.0);
+    let test_source = SimulationTestSource::register_new(source_parallelism, 0..100000, 0.2, 20);
+
+    let mut session = cluster.start_session();
+
+    session.run("set streaming_parallelism = 6").await?;
+    session.run("set sink_decouple = true").await?;
+    session.run(CREATE_SOURCE).await?;
+    session.run(CREATE_SINK).await?;
+    assert_eq!(6, test_sink.parallelism_counter.load(Relaxed));
+
+    test_sink.store.wait_for_err(1).await?;
+
+    session.run(DROP_SINK).await?;
+    session.run(DROP_SOURCE).await?;
+
+    // Due to sink failure isolation, source stream should not be recreated
+    assert_eq!(
+        source_parallelism,
+        test_source.create_stream_count.load(Relaxed)
+    );
+
+    // Sink error should be recorded in rw_event_logs
+    let result = session
+        .run("select * from rw_event_logs where event_type = 'SINK_FAIL'")
+        .await?;
+    assert!(!result.is_empty());
+    println!("Sink fail event logs: {:?}", result);
+
+    Ok(())
+}
diff --git a/src/tests/simulation/tests/integration_tests/sink/utils.rs b/src/tests/simulation/tests/integration_tests/sink/utils.rs
index a134ab1a265ff..bef5bdfa35d0b 100644
--- a/src/tests/simulation/tests/integration_tests/sink/utils.rs
+++ b/src/tests/simulation/tests/integration_tests/sink/utils.rs
@@ -52,6 +52,7 @@ pub struct TestSinkStoreInner {
     pub id_name: HashMap<i32, Vec<String>>,
     pub epochs: Vec<u64>,
     pub checkpoint_count: usize,
+    pub err_count: usize,
 }
 
 #[derive(Clone)]
@@ -66,6 +67,7 @@ impl TestSinkStore {
                 id_name: HashMap::new(),
                 epochs: Vec::new(),
                 checkpoint_count: 0,
+                err_count: 0,
             })),
         }
     }
@@ -99,6 +101,10 @@ impl TestSinkStore {
         self.inner().id_name.len()
     }
 
+    pub fn err_count(&self) -> usize {
+        self.inner().err_count
+    }
+
     pub async fn wait_for_count(&self, count: usize) -> anyhow::Result<()> {
         let mut prev_count = 0;
         let mut has_printed = false;
@@ -122,6 +128,16 @@ impl TestSinkStore {
         }
         Ok(())
     }
+
+    pub async fn wait_for_err(&self, count: usize) -> anyhow::Result<()> {
+        loop {
+            sleep(Duration::from_secs(1)).await;
+            if self.err_count() >= count {
+                break;
+            }
+        }
+        Ok(())
+    }
 }
 
 pub struct TestWriter {
@@ -154,6 +170,7 @@ impl SinkWriter for TestWriter {
     async fn write_batch(&mut self, chunk: StreamChunk) -> risingwave_connector::sink::Result<()> {
         if thread_rng().gen_ratio(self.err_rate.load(Relaxed), u32::MAX) {
             println!("write with err");
+            self.store.inner().err_count += 1;
             return Err(SinkError::Internal(anyhow::anyhow!("fail to write")));
         }
         for (op, row) in chunk.rows() {
diff --git a/src/tests/sqlsmith/src/lib.rs b/src/tests/sqlsmith/src/lib.rs
index 1c31041a7e484..cc16414f73cef 100644
--- a/src/tests/sqlsmith/src/lib.rs
+++ b/src/tests/sqlsmith/src/lib.rs
@@ -276,7 +276,7 @@ CREATE TABLE t3(v1 int, v2 bool, v3 smallint);
                             wildcard_idx: None,
                             constraints: [],
                             with_options: [],
-                            source_schema: None,
+                            format_encode: None,
                             source_watermarks: [],
                             append_only: false,
                             on_conflict: None,
@@ -324,7 +324,7 @@ CREATE TABLE t3(v1 int, v2 bool, v3 smallint);
                             wildcard_idx: None,
                             constraints: [],
                             with_options: [],
-                            source_schema: None,
+                            format_encode: None,
                             source_watermarks: [],
                             append_only: false,
                             on_conflict: None,
@@ -383,7 +383,7 @@ CREATE TABLE t3(v1 int, v2 bool, v3 smallint);
                             wildcard_idx: None,
                             constraints: [],
                             with_options: [],
-                            source_schema: None,
+                            format_encode: None,
                             source_watermarks: [],
                             append_only: false,
                             on_conflict: None,
@@ -518,7 +518,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY
                             wildcard_idx: None,
                             constraints: [],
                             with_options: [],
-                            source_schema: None,
+                            format_encode: None,
                             source_watermarks: [],
                             append_only: false,
                             on_conflict: None,
@@ -573,7 +573,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY
                             wildcard_idx: None,
                             constraints: [],
                             with_options: [],
-                            source_schema: None,
+                            format_encode: None,
                             source_watermarks: [],
                             append_only: false,
                             on_conflict: None,
@@ -635,7 +635,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY
                             wildcard_idx: None,
                             constraints: [],
                             with_options: [],
-                            source_schema: None,
+                            format_encode: None,
                             source_watermarks: [],
                             append_only: false,
                             on_conflict: None,
@@ -715,7 +715,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY
                             wildcard_idx: None,
                             constraints: [],
                             with_options: [],
-                            source_schema: None,
+                            format_encode: None,
                             source_watermarks: [],
                             append_only: false,
                             on_conflict: None,
diff --git a/src/tests/sqlsmith/src/sql_gen/agg.rs b/src/tests/sqlsmith/src/sql_gen/agg.rs
index 177603ddb333a..90f9dad3aaedf 100644
--- a/src/tests/sqlsmith/src/sql_gen/agg.rs
+++ b/src/tests/sqlsmith/src/sql_gen/agg.rs
@@ -24,7 +24,7 @@ use risingwave_sqlparser::ast::{
 use crate::sql_gen::types::AGG_FUNC_TABLE;
 use crate::sql_gen::{SqlGenerator, SqlGeneratorContext};
 
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     pub fn gen_agg(&mut self, ret: &DataType) -> Expr {
         let funcs = match AGG_FUNC_TABLE.get(ret) {
             None => return self.gen_simple_scalar(ret),
diff --git a/src/tests/sqlsmith/src/sql_gen/cast.rs b/src/tests/sqlsmith/src/sql_gen/cast.rs
index 464d6e39d86f3..983333c6ca4f2 100644
--- a/src/tests/sqlsmith/src/sql_gen/cast.rs
+++ b/src/tests/sqlsmith/src/sql_gen/cast.rs
@@ -21,7 +21,7 @@ use risingwave_sqlparser::ast::Expr;
 use crate::sql_gen::types::{data_type_to_ast_data_type, EXPLICIT_CAST_TABLE};
 use crate::sql_gen::{SqlGenerator, SqlGeneratorContext};
 
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     pub(crate) fn gen_explicit_cast(
         &mut self,
         ret: &DataType,
diff --git a/src/tests/sqlsmith/src/sql_gen/expr.rs b/src/tests/sqlsmith/src/sql_gen/expr.rs
index ee4c7ac2ee9ee..4625727f67dca 100644
--- a/src/tests/sqlsmith/src/sql_gen/expr.rs
+++ b/src/tests/sqlsmith/src/sql_gen/expr.rs
@@ -28,7 +28,7 @@ static STRUCT_FIELD_NAMES: [&str; 26] = [
     "t", "u", "v", "w", "x", "y", "z",
 ];
 
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     /// In generating expression, there are two execution modes:
     /// 1) Can have Aggregate expressions (`can_agg` = true)
     ///    We can have aggregate of all bound columns (those present in GROUP BY and otherwise).
diff --git a/src/tests/sqlsmith/src/sql_gen/functions.rs b/src/tests/sqlsmith/src/sql_gen/functions.rs
index 8cd1645ec1f5b..7e78334de9faa 100644
--- a/src/tests/sqlsmith/src/sql_gen/functions.rs
+++ b/src/tests/sqlsmith/src/sql_gen/functions.rs
@@ -25,7 +25,7 @@ use risingwave_sqlparser::ast::{
 use crate::sql_gen::types::{FUNC_TABLE, IMPLICIT_CAST_TABLE, INVARIANT_FUNC_SET};
 use crate::sql_gen::{SqlGenerator, SqlGeneratorContext};
 
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     pub fn gen_func(&mut self, ret: &DataType, context: SqlGeneratorContext) -> Expr {
         match self.rng.gen_bool(0.1) {
             true => self.gen_special_func(ret, context),
diff --git a/src/tests/sqlsmith/src/sql_gen/query.rs b/src/tests/sqlsmith/src/sql_gen/query.rs
index dcead4275c7ad..cfbfdb70cbf8e 100644
--- a/src/tests/sqlsmith/src/sql_gen/query.rs
+++ b/src/tests/sqlsmith/src/sql_gen/query.rs
@@ -30,7 +30,7 @@ use crate::sql_gen::utils::create_table_with_joins_from_table;
 use crate::sql_gen::{Column, SqlGenerator, SqlGeneratorContext, Table};
 
 /// Generators
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     /// Generates query expression and returns its
     /// query schema as well.
     pub(crate) fn gen_query(&mut self) -> (Query, Vec<Column>) {
diff --git a/src/tests/sqlsmith/src/sql_gen/relation.rs b/src/tests/sqlsmith/src/sql_gen/relation.rs
index 6e6db4e40493d..71242febe780c 100644
--- a/src/tests/sqlsmith/src/sql_gen/relation.rs
+++ b/src/tests/sqlsmith/src/sql_gen/relation.rs
@@ -33,7 +33,7 @@ fn create_equi_expr(left: String, right: String) -> Expr {
     create_binary_expr(BinaryOperator::Eq, left, right)
 }
 
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     /// A relation specified in the FROM clause.
     pub(crate) fn gen_from_relation(&mut self) -> (TableWithJoins, Vec<Table>) {
         match self.rng.gen_range(1..=4) {
diff --git a/src/tests/sqlsmith/src/sql_gen/scalar.rs b/src/tests/sqlsmith/src/sql_gen/scalar.rs
index 93cdf6518e73e..62cd7218dcc90 100644
--- a/src/tests/sqlsmith/src/sql_gen/scalar.rs
+++ b/src/tests/sqlsmith/src/sql_gen/scalar.rs
@@ -24,7 +24,7 @@ use risingwave_sqlparser::ast::{Array, DataType as AstDataType, Expr, Value};
 use crate::sql_gen::expr::typed_null;
 use crate::sql_gen::SqlGenerator;
 
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     /// Generates integer scalar expression.
     /// Bound: [start, end).
     /// Type: `DataType`.
diff --git a/src/tests/sqlsmith/src/sql_gen/time_window.rs b/src/tests/sqlsmith/src/sql_gen/time_window.rs
index a4d619b574526..d8e783df5bd11 100644
--- a/src/tests/sqlsmith/src/sql_gen/time_window.rs
+++ b/src/tests/sqlsmith/src/sql_gen/time_window.rs
@@ -22,7 +22,7 @@ use risingwave_sqlparser::ast::{
 use crate::sql_gen::utils::{create_args, create_table_alias};
 use crate::sql_gen::{Column, Expr, SqlGenerator, Table};
 
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     /// Generates time window functions.
     pub(crate) fn gen_time_window_func(&mut self) -> (TableFactor, Table) {
         match self.flip_coin() {
diff --git a/src/tests/sqlsmith/src/sql_gen/utils.rs b/src/tests/sqlsmith/src/sql_gen/utils.rs
index 0e36507e86169..a00d61456bd54 100644
--- a/src/tests/sqlsmith/src/sql_gen/utils.rs
+++ b/src/tests/sqlsmith/src/sql_gen/utils.rs
@@ -25,7 +25,7 @@ use crate::sql_gen::{Column, Expr, Ident, ObjectName, SqlGenerator, Table};
 type Context = (Vec<Column>, Vec<Table>);
 
 /// Context utils
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     pub(crate) fn add_relations_to_context(&mut self, mut tables: Vec<Table>) {
         for rel in &tables {
             let mut bound_columns = rel.get_qualified_columns();
@@ -53,7 +53,7 @@ impl<'a, R: Rng> SqlGenerator<'a, R> {
 }
 
 /// Gen utils
-impl<'a, R: Rng> SqlGenerator<'a, R> {
+impl<R: Rng> SqlGenerator<'_, R> {
     pub(crate) fn gen_table_name_with_prefix(&mut self, prefix: &str) -> String {
         format!("{}_{}", prefix, &self.gen_relation_id())
     }
diff --git a/src/utils/delta_btree_map/src/lib.rs b/src/utils/delta_btree_map/src/lib.rs
index 1f7e1a77190a0..afb49fcda469f 100644
--- a/src/utils/delta_btree_map/src/lib.rs
+++ b/src/utils/delta_btree_map/src/lib.rs
@@ -15,7 +15,7 @@
 #![feature(btree_cursors)]
 
 use std::cmp::Ordering;
-use std::collections::BTreeMap;
+use std::collections::{btree_map, BTreeMap};
 use std::ops::Bound;
 
 use educe::Educe;
@@ -28,6 +28,9 @@ use enum_as_inner::EnumAsInner;
 pub struct DeltaBTreeMap<'a, K: Ord, V> {
     snapshot: &'a BTreeMap<K, V>,
     delta: &'a BTreeMap<K, Change<V>>,
+
+    first_key: Option<&'a K>,
+    last_key: Option<&'a K>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, EnumAsInner)]
@@ -37,8 +40,29 @@ pub enum Change<V> {
 }
 
 impl<'a, K: Ord, V> DeltaBTreeMap<'a, K, V> {
+    /// Create a new [`DeltaBTreeMap`] from the given snapshot and delta.
+    /// Best case time complexity: O(1), worst case time complexity: O(m), where m is `delta.len()`.
     pub fn new(snapshot: &'a BTreeMap<K, V>, delta: &'a BTreeMap<K, Change<V>>) -> Self {
-        Self { snapshot, delta }
+        let first_key = {
+            let cursor = CursorWithDelta {
+                ss_cursor: snapshot.lower_bound(Bound::Unbounded),
+                dt_cursor: delta.lower_bound(Bound::Unbounded),
+            };
+            cursor.peek_next().map(|(key, _)| key)
+        };
+        let last_key = {
+            let cursor = CursorWithDelta {
+                ss_cursor: snapshot.upper_bound(Bound::Unbounded),
+                dt_cursor: delta.upper_bound(Bound::Unbounded),
+            };
+            cursor.peek_prev().map(|(key, _)| key)
+        };
+        Self {
+            snapshot,
+            delta,
+            first_key,
+            last_key,
+        }
     }
 
     /// Get a reference to the snapshot.
@@ -51,238 +75,170 @@ impl<'a, K: Ord, V> DeltaBTreeMap<'a, K, V> {
         self.delta
     }
 
-    /// Get the first key in the updated version of the snapshot.
+    /// Get the first key in the updated version of the snapshot. Complexity: O(1).
     pub fn first_key(&self) -> Option<&'a K> {
-        let cursor = CursorWithDelta {
-            snapshot: self.snapshot,
-            delta: self.delta,
-            curr_key_value: None,
-        };
-        cursor.peek_next().map(|(key, _)| key)
+        self.first_key
     }
 
-    /// Get the last key in the updated version of the snapshot.
+    /// Get the last key in the updated version of the snapshot. Complexity: O(1).
     pub fn last_key(&self) -> Option<&'a K> {
-        let cursor = CursorWithDelta {
-            snapshot: self.snapshot,
-            delta: self.delta,
-            curr_key_value: None,
-        };
-        cursor.peek_prev().map(|(key, _)| key)
+        self.last_key
     }
 
-    /// Get a [`CursorWithDelta`] pointing to the element corresponding to the given key.
+    /// Get a [`CursorWithDelta`] pointing at the gap before the given given key.
     /// If the given key is not found in either the snapshot or the delta, `None` is returned.
-    pub fn find(&self, key: &K) -> Option<CursorWithDelta<'a, K, V>> {
-        let ss_cursor = self.snapshot.lower_bound(Bound::Included(key));
-        let dt_cursor = self.delta.lower_bound(Bound::Included(key));
-        let ss_cursor_kv = ss_cursor.peek_next();
-        let dt_cursor_kv = dt_cursor.peek_next();
-        let curr_key_value = if dt_cursor_kv.map(|(k, _)| k) == Some(key) {
-            match dt_cursor_kv.unwrap() {
-                (key, Change::Insert(value)) => (key, value),
-                (_key, Change::Delete) => {
-                    // the key is deleted
-                    return None;
-                }
-            }
-        } else if ss_cursor_kv.map(|(k, _)| k) == Some(key) {
-            ss_cursor_kv.unwrap()
-        } else {
-            // the key doesn't exist
+    pub fn before(&self, key: &K) -> Option<CursorWithDelta<'a, K, V>> {
+        let cursor = self.lower_bound(Bound::Included(key));
+        if cursor.peek_next().map(|(k, _)| k) != Some(key) {
             return None;
-        };
-        Some(CursorWithDelta {
-            snapshot: self.snapshot,
-            delta: self.delta,
-            curr_key_value: Some(curr_key_value),
-        })
+        }
+        Some(cursor)
+    }
+
+    /// Get a [`CursorWithDelta`] pointing at the gap after the given given key.
+    /// If the given key is not found in either the snapshot or the delta, `None` is returned.
+    pub fn after(&self, key: &K) -> Option<CursorWithDelta<'a, K, V>> {
+        let cursor = self.upper_bound(Bound::Included(key));
+        if cursor.peek_prev().map(|(k, _)| k) != Some(key) {
+            return None;
+        }
+        Some(cursor)
     }
 
-    /// Get a [`CursorWithDelta`] pointing to the first element that is above the given bound.
+    /// Get a [`CursorWithDelta`] pointing at the gap before the smallest key greater than the given bound.
     pub fn lower_bound(&self, bound: Bound<&K>) -> CursorWithDelta<'a, K, V> {
-        // the implementation is very similar to `CursorWithDelta::peek_next`
-        let mut ss_cursor = self.snapshot.lower_bound(bound);
-        let mut dt_cursor = self.delta.lower_bound(bound);
-        let next_ss_entry = || ss_cursor.next();
-        let next_dt_entry = || dt_cursor.next();
-        let curr_key_value =
-            CursorWithDelta::peek_impl(PeekDirection::Next, next_ss_entry, next_dt_entry);
+        let ss_cursor = self.snapshot.lower_bound(bound);
+        let dt_cursor = self.delta.lower_bound(bound);
         CursorWithDelta {
-            snapshot: self.snapshot,
-            delta: self.delta,
-            curr_key_value,
+            ss_cursor,
+            dt_cursor,
         }
     }
 
-    /// Get a [`CursorWithDelta`] pointing to the first element that is below the given bound.
+    /// Get a [`CursorWithDelta`] pointing at the gap after the greatest key smaller than the given bound.
     pub fn upper_bound(&self, bound: Bound<&K>) -> CursorWithDelta<'a, K, V> {
-        // the implementation is very similar to `CursorWithDelta::peek_prev`
-        let mut ss_cursor = self.snapshot.upper_bound(bound);
-        let mut dt_cursor = self.delta.upper_bound(bound);
-        let prev_ss_entry = || ss_cursor.prev();
-        let prev_dt_entry = || dt_cursor.prev();
-        let curr_key_value =
-            CursorWithDelta::peek_impl(PeekDirection::Prev, prev_ss_entry, prev_dt_entry);
+        let ss_cursor = self.snapshot.upper_bound(bound);
+        let dt_cursor = self.delta.upper_bound(bound);
         CursorWithDelta {
-            snapshot: self.snapshot,
-            delta: self.delta,
-            curr_key_value,
+            ss_cursor,
+            dt_cursor,
         }
     }
 }
 
 /// Cursor that can iterate back and forth over the updated version of the snapshot.
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+///
+/// A cursor always points at the gap of items in the map. For example:
+///
+/// ```text
+/// | Foo | Bar |
+/// ^     ^     ^
+/// 1     2     3
+/// ```
+///
+/// The cursor can be at position 1, 2, or 3.
+/// If it's at position 1, `peek_prev` will return `None`, and `peek_next` will return `Foo`.
+/// If it's at position 3, `peek_prev` will return `Bar`, and `peek_next` will return `None`.
+#[derive(Debug, Clone)]
 pub struct CursorWithDelta<'a, K: Ord, V> {
-    snapshot: &'a BTreeMap<K, V>,
-    delta: &'a BTreeMap<K, Change<V>>,
-    curr_key_value: Option<(&'a K, &'a V)>,
+    ss_cursor: btree_map::Cursor<'a, K, V>,
+    dt_cursor: btree_map::Cursor<'a, K, Change<V>>,
 }
 
-/// Type of cursor position. [`PositionType::Ghost`] is a special position between the first and
-/// the last item, where the key and value are `None`.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumAsInner)]
-pub enum PositionType {
-    Ghost,
-    Snapshot,
-    DeltaUpdate,
-    DeltaInsert,
-}
+impl<'a, K: Ord, V> CursorWithDelta<'a, K, V> {
+    pub fn peek_prev(&self) -> Option<(&'a K, &'a V)> {
+        self.peek::<false /* PREV */>()
+    }
 
-#[derive(PartialEq, Eq)]
-enum PeekDirection {
-    Next,
-    Prev,
-}
+    pub fn peek_next(&self) -> Option<(&'a K, &'a V)> {
+        self.peek::<true /* NEXT */>()
+    }
 
-impl<'a, K: Ord, V> CursorWithDelta<'a, K, V> {
-    /// Get the cursor position type.
-    pub fn position(&self) -> PositionType {
-        let Some((key, _)) = self.curr_key_value else {
-            return PositionType::Ghost;
-        };
-        if self.delta.contains_key(key) {
-            assert!(matches!(self.delta.get(key).unwrap(), Change::Insert(_)));
-            if self.snapshot.contains_key(key) {
-                PositionType::DeltaUpdate
-            } else {
-                PositionType::DeltaInsert
-            }
-        } else {
-            assert!(self.snapshot.contains_key(key));
-            PositionType::Snapshot
-        }
+    pub fn prev(&mut self) -> Option<(&'a K, &'a V)> {
+        self.r#move::<false /* PREV */>()
     }
 
-    /// Get the key pointed by the cursor.
-    pub fn key(&self) -> Option<&'a K> {
-        self.curr_key_value.map(|(k, _)| k)
+    #[allow(clippy::should_implement_trait)]
+    pub fn next(&mut self) -> Option<(&'a K, &'a V)> {
+        self.r#move::<true /* NEXT */>()
     }
 
-    /// Get the value pointed by the cursor.
-    pub fn value(&self) -> Option<&'a V> {
-        self.curr_key_value.map(|(_, v)| v)
+    fn peek<const NEXT: bool>(&self) -> Option<(&'a K, &'a V)> {
+        let mut ss_cursor = self.ss_cursor.clone();
+        let mut dt_cursor = self.dt_cursor.clone();
+        let res = Self::move_impl::<NEXT>(&mut ss_cursor, &mut dt_cursor);
+        res
     }
 
-    /// Get the key-value pair pointed by the cursor.
-    pub fn key_value(&self) -> Option<(&'a K, &'a V)> {
-        self.curr_key_value
+    fn r#move<const NEXT: bool>(&mut self) -> Option<(&'a K, &'a V)> {
+        let mut ss_cursor = self.ss_cursor.clone();
+        let mut dt_cursor = self.dt_cursor.clone();
+        let res = Self::move_impl::<NEXT>(&mut ss_cursor, &mut dt_cursor);
+        self.ss_cursor = ss_cursor;
+        self.dt_cursor = dt_cursor;
+        res
     }
 
-    fn peek_impl(
-        direction: PeekDirection,
-        mut next_ss_entry: impl FnMut() -> Option<(&'a K, &'a V)>,
-        mut next_dt_entry: impl FnMut() -> Option<(&'a K, &'a Change<V>)>,
+    fn move_impl<const NEXT: bool>(
+        ss_cursor: &mut btree_map::Cursor<'a, K, V>,
+        dt_cursor: &mut btree_map::Cursor<'a, K, Change<V>>,
     ) -> Option<(&'a K, &'a V)> {
         loop {
-            match (next_ss_entry(), next_dt_entry()) {
+            let ss_peek = if NEXT {
+                ss_cursor.peek_next()
+            } else {
+                ss_cursor.peek_prev()
+            };
+            let dt_peek = if NEXT {
+                dt_cursor.peek_next()
+            } else {
+                dt_cursor.peek_prev()
+            };
+
+            let in_delta = match (ss_peek, dt_peek) {
                 (None, None) => return None,
-                (None, Some((key, change))) => return Some((key, change.as_insert().unwrap())),
-                (Some((key, value)), None) => return Some((key, value)),
-                (Some((ss_key, ss_value)), Some((dt_key, dt_change))) => match ss_key.cmp(dt_key) {
-                    Ordering::Less => {
-                        if direction == PeekDirection::Next {
-                            return Some((ss_key, ss_value));
+                (None, Some(_)) => true,
+                (Some(_), None) => false,
+                (Some((ss_key, _)), Some((dt_key, dt_change))) => match ss_key.cmp(dt_key) {
+                    Ordering::Less => !NEXT,   // if NEXT { in snapshot } else { in delta }
+                    Ordering::Greater => NEXT, // if NEXT { in delta } else { in snapshot }
+                    Ordering::Equal => {
+                        if NEXT {
+                            ss_cursor.next().unwrap();
                         } else {
-                            return Some((dt_key, dt_change.as_insert().unwrap()));
+                            ss_cursor.prev().unwrap();
                         }
-                    }
-                    Ordering::Greater => {
-                        if direction == PeekDirection::Next {
-                            return Some((dt_key, dt_change.as_insert().unwrap()));
-                        } else {
-                            return Some((ss_key, ss_value));
+                        match dt_change {
+                            Change::Insert(_) => true, // in delta
+                            Change::Delete => {
+                                if NEXT {
+                                    dt_cursor.next().unwrap();
+                                } else {
+                                    dt_cursor.prev().unwrap();
+                                }
+                                continue;
+                            }
                         }
                     }
-                    Ordering::Equal => match dt_change {
-                        Change::Insert(v) => return Some((ss_key, v)),
-                        Change::Delete => continue,
-                    },
                 },
+            };
+
+            if in_delta {
+                let (key, change) = if NEXT {
+                    dt_cursor.next().unwrap()
+                } else {
+                    dt_cursor.prev().unwrap()
+                };
+                return Some((key, change.as_insert().unwrap()));
+            } else {
+                return if NEXT {
+                    ss_cursor.next()
+                } else {
+                    ss_cursor.prev()
+                };
             }
         }
     }
-
-    /// Peek the next key-value pair.
-    pub fn peek_next(&self) -> Option<(&'a K, &'a V)> {
-        if let Some(key) = self.key() {
-            let mut ss_cursor = self.snapshot.lower_bound(Bound::Included(key));
-            let mut dt_cursor = self.delta.lower_bound(Bound::Included(key));
-            // either one of `ss_cursor.key()` and `dt_cursor.key()` == `Some(key)`, or both are
-            if ss_cursor.peek_next().map(|(k, _)| k) == Some(key) {
-                ss_cursor.next();
-            }
-            if dt_cursor.peek_next().map(|(k, _)| k) == Some(key) {
-                dt_cursor.next();
-            }
-            let next_ss_entry = || ss_cursor.next();
-            let next_dt_entry = || dt_cursor.next();
-            Self::peek_impl(PeekDirection::Next, next_ss_entry, next_dt_entry)
-        } else {
-            // we are at the ghost position, now let's go back to the beginning
-            let mut ss_iter = self.snapshot.iter();
-            let mut dt_iter = self.delta.iter();
-            Self::peek_impl(PeekDirection::Next, || ss_iter.next(), || dt_iter.next())
-        }
-    }
-
-    /// Peek the previous key-value pair.
-    pub fn peek_prev(&self) -> Option<(&'a K, &'a V)> {
-        if let Some(key) = self.key() {
-            let mut ss_cursor = self.snapshot.upper_bound(Bound::Included(key));
-            let mut dt_cursor = self.delta.upper_bound(Bound::Included(key));
-            // either one of `ss_cursor.key()` and `dt_cursor.key()` == `Some(key)`, or both are
-            if ss_cursor.peek_prev().map(|(k, _)| k) == Some(key) {
-                ss_cursor.prev();
-            }
-            if dt_cursor.peek_prev().map(|(k, _)| k) == Some(key) {
-                dt_cursor.prev();
-            }
-            let next_ss_entry = || ss_cursor.prev();
-            let next_dt_entry = || dt_cursor.prev();
-            Self::peek_impl(PeekDirection::Prev, next_ss_entry, next_dt_entry)
-        } else {
-            // we are at the ghost position, now let's go back to the end
-            let mut ss_iter = self.snapshot.iter();
-            let mut dt_iter = self.delta.iter();
-            Self::peek_impl(
-                PeekDirection::Prev,
-                || ss_iter.next_back(),
-                || dt_iter.next_back(),
-            )
-        }
-    }
-
-    /// Move the cursor to the next position.
-    pub fn move_next(&mut self) {
-        self.curr_key_value = self.peek_next();
-    }
-
-    /// Move the cursor to the previous position.
-    pub fn move_prev(&mut self) {
-        self.curr_key_value = self.peek_prev();
-    }
 }
 
 #[cfg(test)]
@@ -297,9 +253,10 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), None);
         assert_eq!(delta_map.last_key(), None);
-        assert_eq!(delta_map.find(&1), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).key(), None);
-        assert_eq!(delta_map.upper_bound(Bound::Included(&1)).key(), None);
+        assert!(delta_map.before(&1).is_none());
+        assert!(delta_map.after(&1).is_none());
+        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).peek_next(), None);
+        assert_eq!(delta_map.upper_bound(Bound::Included(&1)).peek_prev(), None);
 
         let mut map = BTreeMap::new();
         map.insert(1, "1");
@@ -310,9 +267,9 @@ mod tests {
         let delta_map = DeltaBTreeMap::new(&map, &delta);
         assert_eq!(delta_map.first_key(), None);
         assert_eq!(delta_map.last_key(), None);
-        assert_eq!(delta_map.find(&1), None);
-        assert_eq!(delta_map.find(&2), None);
-        assert_eq!(delta_map.find(&3), None);
+        assert!(delta_map.before(&1).is_none());
+        assert!(delta_map.before(&2).is_none());
+        assert!(delta_map.before(&3).is_none());
     }
 
     #[test]
@@ -326,40 +283,46 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&5));
-        assert_eq!(delta_map.find(&100), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).key(), Some(&1));
-        assert_eq!(delta_map.lower_bound(Bound::Excluded(&3)).key(), Some(&5));
-        assert_eq!(delta_map.upper_bound(Bound::Included(&1)).key(), Some(&1));
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&3)).key(), Some(&2));
-
-        let mut cursor = delta_map.find(&2).unwrap();
-        assert_eq!(cursor.position(), PositionType::Snapshot);
-        assert_eq!(cursor.key(), Some(&2));
-        assert_eq!(cursor.value(), Some(&"2"));
-        assert_eq!(cursor.key_value(), Some((&2, &"2")));
-        assert_eq!(cursor.peek_next(), Some((&5, &"5")));
-        assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
-        cursor.move_next();
-        assert_eq!(cursor.key(), Some(&5));
-        assert_eq!(cursor.value(), Some(&"5"));
-        cursor.move_next();
-        assert_eq!(cursor.position(), PositionType::Ghost);
-        assert_eq!(cursor.key(), None);
-        assert_eq!(cursor.value(), None);
-        cursor.move_prev();
-        assert_eq!(cursor.key(), Some(&5));
-        assert_eq!(cursor.value(), Some(&"5"));
-        cursor.move_prev();
-        cursor.move_prev();
-        assert_eq!(cursor.key(), Some(&1));
-        assert_eq!(cursor.value(), Some(&"1"));
-        assert_eq!(cursor.peek_prev(), None);
+        assert!(delta_map.before(&100).is_none());
+        assert_eq!(
+            delta_map.lower_bound(Bound::Included(&1)).peek_next(),
+            Some((&1, &"1"))
+        );
+        assert_eq!(
+            delta_map.lower_bound(Bound::Excluded(&3)).peek_next(),
+            Some((&5, &"5"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Included(&1)).peek_prev(),
+            Some((&1, &"1"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Excluded(&3)).peek_prev(),
+            Some((&2, &"2"))
+        );
+
+        let mut cursor = delta_map.before(&2).unwrap();
         assert_eq!(cursor.peek_next(), Some((&2, &"2")));
-        cursor.move_prev();
-        assert_eq!(cursor.key(), None);
-        assert_eq!(cursor.value(), None);
+        assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
+        let (key, value) = cursor.next().unwrap();
+        assert_eq!(key, &2);
+        assert_eq!(value, &"2");
+        assert_eq!(cursor.peek_next(), Some((&5, &"5")));
+        assert_eq!(cursor.peek_prev(), Some((&2, &"2")));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), None);
+        cursor.next();
+        assert_eq!(cursor.peek_next(), None);
         assert_eq!(cursor.peek_prev(), Some((&5, &"5")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&5, &"5")));
+        cursor.prev();
+        cursor.prev();
         assert_eq!(cursor.peek_next(), Some((&1, &"1")));
+        assert_eq!(cursor.peek_prev(), None);
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1")));
+        assert_eq!(cursor.peek_prev(), None);
     }
 
     #[test]
@@ -372,35 +335,42 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&2));
-        assert_eq!(delta_map.find(&100), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).key(), Some(&1));
-        assert_eq!(delta_map.lower_bound(Bound::Excluded(&1)).key(), Some(&2));
-        assert_eq!(delta_map.upper_bound(Bound::Included(&1)).key(), Some(&1));
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&10)).key(), Some(&2));
-
-        let mut cursor = delta_map.find(&2).unwrap();
-        assert_eq!(cursor.position(), PositionType::DeltaInsert);
-        assert_eq!(cursor.key(), Some(&2));
-        assert_eq!(cursor.value(), Some(&"2"));
-        assert_eq!(cursor.key_value(), Some((&2, &"2")));
+        assert!(delta_map.before(&100).is_none());
+        assert_eq!(
+            delta_map.lower_bound(Bound::Included(&1)).peek_next(),
+            Some((&1, &"1"))
+        );
+        assert_eq!(
+            delta_map.lower_bound(Bound::Excluded(&1)).peek_next(),
+            Some((&2, &"2"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Included(&1)).peek_prev(),
+            Some((&1, &"1"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Excluded(&10)).peek_prev(),
+            Some((&2, &"2"))
+        );
+
+        let mut cursor = delta_map.before(&2).unwrap();
+        assert_eq!(cursor.peek_next(), Some((&2, &"2")));
+        assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), None);
+        assert_eq!(cursor.peek_prev(), Some((&2, &"2")));
+        cursor.next();
         assert_eq!(cursor.peek_next(), None);
+        assert_eq!(cursor.peek_prev(), Some((&2, &"2")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&2, &"2")));
         assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
-        cursor.move_next();
-        assert_eq!(cursor.key(), None);
-        assert_eq!(cursor.value(), None);
-        cursor.move_prev();
-        assert_eq!(cursor.key(), Some(&2));
-        assert_eq!(cursor.value(), Some(&"2"));
-        cursor.move_prev();
-        assert_eq!(cursor.key(), Some(&1));
-        assert_eq!(cursor.value(), Some(&"1"));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1")));
         assert_eq!(cursor.peek_prev(), None);
-        assert_eq!(cursor.peek_next(), Some((&2, &"2")));
-        cursor.move_prev();
-        assert_eq!(cursor.key(), None);
-        assert_eq!(cursor.value(), None);
-        assert_eq!(cursor.peek_prev(), Some((&2, &"2")));
+        cursor.prev();
         assert_eq!(cursor.peek_next(), Some((&1, &"1")));
+        assert_eq!(cursor.peek_prev(), None);
     }
 
     #[test]
@@ -414,26 +384,37 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), Some(&3));
         assert_eq!(delta_map.last_key(), Some(&3));
-        assert_eq!(delta_map.find(&1), None);
-        assert_eq!(delta_map.find(&2), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).key(), Some(&3));
-        assert_eq!(delta_map.lower_bound(Bound::Excluded(&0)).key(), Some(&3));
-        assert_eq!(delta_map.upper_bound(Bound::Included(&1)).key(), None);
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&10)).key(), Some(&3));
-
-        let mut cursor = delta_map.find(&3).unwrap();
-        assert_eq!(cursor.position(), PositionType::Snapshot);
-        assert_eq!(cursor.key(), Some(&3));
-        assert_eq!(cursor.value(), Some(&"3"));
-        assert_eq!(cursor.key_value(), Some((&3, &"3")));
+        assert!(delta_map.before(&1).is_none());
+        assert!(delta_map.before(&2).is_none());
+        assert_eq!(
+            delta_map.lower_bound(Bound::Included(&1)).peek_next(),
+            Some((&3, &"3"))
+        );
+        assert_eq!(
+            delta_map.lower_bound(Bound::Excluded(&0)).peek_next(),
+            Some((&3, &"3"))
+        );
+        assert_eq!(delta_map.upper_bound(Bound::Included(&1)).peek_prev(), None);
+        assert_eq!(
+            delta_map.upper_bound(Bound::Excluded(&10)).peek_prev(),
+            Some((&3, &"3"))
+        );
+
+        let mut cursor = delta_map.before(&3).unwrap();
+        assert_eq!(cursor.peek_next(), Some((&3, &"3")));
+        assert_eq!(cursor.peek_prev(), None);
+        cursor.next();
         assert_eq!(cursor.peek_next(), None);
+        assert_eq!(cursor.peek_prev(), Some((&3, &"3")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&3, &"3")));
+        assert_eq!(cursor.peek_prev(), None);
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&3, &"3")));
         assert_eq!(cursor.peek_prev(), None);
-        cursor.move_next();
-        assert_eq!(cursor.key(), None);
-        assert_eq!(cursor.value(), None);
-        cursor.move_prev();
-        assert_eq!(cursor.key(), Some(&3));
-        assert_eq!(cursor.value(), Some(&"3"));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), None);
+        assert_eq!(cursor.peek_prev(), Some((&3, &"3")));
     }
 
     #[test]
@@ -447,15 +428,30 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&1));
-        assert_eq!(delta_map.find(&2), None);
-        assert_eq!(delta_map.find(&3), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).key(), Some(&1));
-        assert_eq!(delta_map.lower_bound(Bound::Excluded(&1)).key(), None);
-        assert_eq!(delta_map.upper_bound(Bound::Included(&3)).key(), Some(&1));
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&3)).key(), Some(&1));
-
-        let cursor = delta_map.find(&1).unwrap();
-        assert_eq!(cursor.position(), PositionType::Snapshot);
+        assert!(delta_map.before(&2).is_none());
+        assert!(delta_map.before(&3).is_none());
+        assert_eq!(
+            delta_map.lower_bound(Bound::Included(&1)).peek_next(),
+            Some((&1, &"1"))
+        );
+        assert_eq!(delta_map.lower_bound(Bound::Excluded(&1)).peek_next(), None);
+        assert_eq!(
+            delta_map.upper_bound(Bound::Included(&3)).peek_prev(),
+            Some((&1, &"1"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Excluded(&3)).peek_prev(),
+            Some((&1, &"1"))
+        );
+
+        let mut cursor = delta_map.before(&1).unwrap();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1")));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), None);
+        assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1")));
+        assert_eq!(cursor.peek_prev(), None);
     }
 
     #[test]
@@ -470,11 +466,11 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), None);
         assert_eq!(delta_map.last_key(), None);
-        assert_eq!(delta_map.find(&1), None);
-        assert_eq!(delta_map.find(&2), None);
-        assert_eq!(delta_map.find(&3), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).key(), None);
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&3)).key(), None);
+        assert!(delta_map.before(&1).is_none());
+        assert!(delta_map.before(&2).is_none());
+        assert!(delta_map.before(&3).is_none());
+        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).peek_next(), None);
+        assert_eq!(delta_map.upper_bound(Bound::Excluded(&3)).peek_prev(), None);
     }
 
     #[test]
@@ -488,22 +484,30 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&3));
-        assert_eq!(delta_map.find(&10), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&1)).key(), Some(&1));
-        assert_eq!(delta_map.lower_bound(Bound::Excluded(&1)).key(), Some(&2));
-        assert_eq!(delta_map.upper_bound(Bound::Included(&2)).key(), Some(&2));
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&2)).key(), Some(&1));
-
-        let mut cursor = delta_map.find(&2).unwrap();
-        assert_eq!(cursor.position(), PositionType::DeltaInsert);
-        assert_eq!(cursor.key(), Some(&2));
-        assert_eq!(cursor.value(), Some(&"2"));
-        assert_eq!(cursor.key_value(), Some((&2, &"2")));
-        assert_eq!(cursor.peek_next(), Some((&3, &"3")));
+        assert!(delta_map.before(&10).is_none());
+        assert_eq!(
+            delta_map.lower_bound(Bound::Included(&1)).peek_next(),
+            Some((&1, &"1"))
+        );
+        assert_eq!(
+            delta_map.lower_bound(Bound::Excluded(&1)).peek_next(),
+            Some((&2, &"2"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Included(&2)).peek_prev(),
+            Some((&2, &"2"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Excluded(&2)).peek_prev(),
+            Some((&1, &"1"))
+        );
+
+        let mut cursor = delta_map.before(&2).unwrap();
+        assert_eq!(cursor.peek_next(), Some((&2, &"2")));
         assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
-        cursor.move_next();
-        assert_eq!(cursor.key(), Some(&3));
-        assert_eq!(cursor.value(), Some(&"3"));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), Some((&3, &"3")));
+        assert_eq!(cursor.peek_prev(), Some((&2, &"2")));
     }
 
     #[test]
@@ -518,19 +522,15 @@ mod tests {
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&3));
 
-        let mut cursor = delta_map.find(&1).unwrap();
-        assert_eq!(cursor.position(), PositionType::DeltaUpdate);
-        assert_eq!(cursor.key(), Some(&1));
-        assert_eq!(cursor.value(), Some(&"1 new"));
-        assert_eq!(cursor.key_value(), Some((&1, &"1 new")));
+        let mut cursor = delta_map.before(&1).unwrap();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1 new")));
+        assert_eq!(cursor.peek_prev(), None);
+        cursor.next();
         assert_eq!(cursor.peek_next(), Some((&3, &"3")));
+        assert_eq!(cursor.peek_prev(), Some((&1, &"1 new")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1 new")));
         assert_eq!(cursor.peek_prev(), None);
-        cursor.move_next();
-        assert_eq!(cursor.key(), Some(&3));
-        assert_eq!(cursor.value(), Some(&"3"));
-        cursor.move_prev();
-        assert_eq!(cursor.key(), Some(&1));
-        assert_eq!(cursor.value(), Some(&"1 new"));
     }
 
     #[test]
@@ -545,19 +545,15 @@ mod tests {
         assert_eq!(delta_map.first_key(), Some(&1));
         assert_eq!(delta_map.last_key(), Some(&3));
 
-        let mut cursor = delta_map.find(&3).unwrap();
-        assert_eq!(cursor.position(), PositionType::DeltaUpdate);
-        assert_eq!(cursor.key(), Some(&3));
-        assert_eq!(cursor.value(), Some(&"3 new"));
-        assert_eq!(cursor.key_value(), Some((&3, &"3 new")));
+        let mut cursor = delta_map.before(&3).unwrap();
+        assert_eq!(cursor.peek_next(), Some((&3, &"3 new")));
+        assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
+        cursor.next();
         assert_eq!(cursor.peek_next(), None);
+        assert_eq!(cursor.peek_prev(), Some((&3, &"3 new")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&3, &"3 new")));
         assert_eq!(cursor.peek_prev(), Some((&1, &"1")));
-        cursor.move_next();
-        assert_eq!(cursor.key(), None);
-        assert_eq!(cursor.value(), None);
-        cursor.move_prev();
-        assert_eq!(cursor.key(), Some(&3));
-        assert_eq!(cursor.value(), Some(&"3 new"));
     }
 
     #[test]
@@ -575,50 +571,58 @@ mod tests {
 
         assert_eq!(delta_map.first_key(), Some(&0));
         assert_eq!(delta_map.last_key(), Some(&4));
-        assert_eq!(delta_map.find(&-1), None);
-        assert_eq!(delta_map.find(&3), None);
-        assert_eq!(delta_map.find(&10), None);
-        assert_eq!(delta_map.lower_bound(Bound::Included(&0)).key(), Some(&0));
-        assert_eq!(delta_map.lower_bound(Bound::Excluded(&0)).key(), Some(&1));
-        assert_eq!(delta_map.lower_bound(Bound::Included(&3)).key(), Some(&4));
-        assert_eq!(delta_map.upper_bound(Bound::Included(&5)).key(), Some(&4));
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&4)).key(), Some(&2));
-        assert_eq!(delta_map.upper_bound(Bound::Excluded(&2)).key(), Some(&1));
-
-        let mut cursor = delta_map.find(&0).unwrap();
-        assert_eq!(cursor.position(), PositionType::DeltaInsert);
-        assert_eq!(cursor.key_value(), Some((&0, &"0")));
-        cursor.move_next();
-        assert_eq!(cursor.position(), PositionType::DeltaUpdate);
-        assert_eq!(cursor.key_value(), Some((&1, &"1 new")));
-        cursor.move_next();
-        assert_eq!(cursor.position(), PositionType::Snapshot);
-        assert_eq!(cursor.key_value(), Some((&2, &"2")));
-        cursor.move_next();
-        assert_eq!(cursor.position(), PositionType::DeltaInsert);
-        assert_eq!(cursor.key_value(), Some((&4, &"4")));
-        cursor.move_next();
-        assert_eq!(cursor.position(), PositionType::Ghost);
-        assert_eq!(cursor.key_value(), None);
-        cursor.move_next();
-        assert_eq!(cursor.position(), PositionType::DeltaInsert);
-        assert_eq!(cursor.key_value(), Some((&0, &"0")));
-        cursor.move_prev();
-        assert_eq!(cursor.position(), PositionType::Ghost);
-        cursor.move_prev();
-        assert_eq!(cursor.position(), PositionType::DeltaInsert);
-        assert_eq!(cursor.key_value(), Some((&4, &"4")));
-        cursor.move_prev();
-        assert_eq!(cursor.position(), PositionType::Snapshot);
-        assert_eq!(cursor.key_value(), Some((&2, &"2")));
-        cursor.move_prev();
-        assert_eq!(cursor.position(), PositionType::DeltaUpdate);
-        assert_eq!(cursor.key_value(), Some((&1, &"1 new")));
-        cursor.move_prev();
-        assert_eq!(cursor.position(), PositionType::DeltaInsert);
-        assert_eq!(cursor.key_value(), Some((&0, &"0")));
-        cursor.move_prev();
-        assert_eq!(cursor.position(), PositionType::Ghost);
+        assert!(delta_map.before(&-1).is_none());
+        assert!(delta_map.before(&3).is_none());
+        assert!(delta_map.before(&10).is_none());
+        assert_eq!(
+            delta_map.lower_bound(Bound::Included(&0)).peek_next(),
+            Some((&0, &"0"))
+        );
+        assert_eq!(
+            delta_map.lower_bound(Bound::Excluded(&0)).peek_next(),
+            Some((&1, &"1 new"))
+        );
+        assert_eq!(
+            delta_map.lower_bound(Bound::Included(&3)).peek_next(),
+            Some((&4, &"4"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Included(&5)).peek_prev(),
+            Some((&4, &"4"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Excluded(&4)).peek_prev(),
+            Some((&2, &"2"))
+        );
+        assert_eq!(
+            delta_map.upper_bound(Bound::Excluded(&2)).peek_prev(),
+            Some((&1, &"1 new"))
+        );
+
+        let mut cursor = delta_map.before(&0).unwrap();
+        assert_eq!(cursor.peek_next(), Some((&0, &"0")));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1 new")));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), Some((&2, &"2")));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), Some((&4, &"4")));
+        cursor.next();
+        assert_eq!(cursor.peek_next(), None);
+        cursor.next();
+        assert_eq!(cursor.peek_next(), None);
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&4, &"4")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&2, &"2")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&1, &"1 new")));
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&0, &"0")));
+        assert_eq!(cursor.peek_prev(), None);
+        cursor.prev();
+        assert_eq!(cursor.peek_next(), Some((&0, &"0")));
+        assert_eq!(cursor.peek_prev(), None);
     }
 
     #[test]
@@ -640,19 +644,17 @@ mod tests {
         assert_eq!(delta_map.first_key(), Some(&0));
         assert_eq!(delta_map.last_key(), Some(&3));
 
-        let mut cursor = delta_map.find(&0).unwrap();
+        let mut cursor = delta_map.before(&0).unwrap();
         let mut res = vec![];
-        while let Some((k, v)) = cursor.key_value() {
+        while let Some((k, v)) = cursor.next() {
             res.push((*k, *v));
-            cursor.move_next();
         }
         assert_eq!(res, vec![(0, "0"), (1, "1 new"), (3, "3")]);
 
-        let mut cursor = delta_map.find(&3).unwrap();
+        let mut cursor = delta_map.after(&3).unwrap();
         let mut res = vec![];
-        while let Some((k, v)) = cursor.key_value() {
+        while let Some((k, v)) = cursor.prev() {
             res.push((*k, *v));
-            cursor.move_prev();
         }
         assert_eq!(res, vec![(3, "3"), (1, "1 new"), (0, "0")]);
     }
diff --git a/src/utils/pgwire/src/pg_message.rs b/src/utils/pgwire/src/pg_message.rs
index 00c9a09b2eda6..58a1cccaf562c 100644
--- a/src/utils/pgwire/src/pg_message.rs
+++ b/src/utils/pgwire/src/pg_message.rs
@@ -426,7 +426,7 @@ pub enum TransactionStatus {
     InFailedTransaction,
 }
 
-impl<'a> BeMessage<'a> {
+impl BeMessage<'_> {
     /// Write message to the given buf.
     pub fn write(buf: &mut BytesMut, message: &BeMessage<'_>) -> Result<()> {
         match message {
diff --git a/src/utils/pgwire/src/pg_protocol.rs b/src/utils/pgwire/src/pg_protocol.rs
index 630feb1ebd28c..5a2120b91f3d1 100644
--- a/src/utils/pgwire/src/pg_protocol.rs
+++ b/src/utils/pgwire/src/pg_protocol.rs
@@ -1190,7 +1190,7 @@ pub mod truncated_fmt {
         finished: bool,
         f: &'a mut Formatter<'b>,
     }
-    impl<'a, 'b> Write for TruncatedFormatter<'a, 'b> {
+    impl Write for TruncatedFormatter<'_, '_> {
         fn write_str(&mut self, s: &str) -> Result {
             if self.finished {
                 return Ok(());
@@ -1212,7 +1212,7 @@ pub mod truncated_fmt {
 
     pub struct TruncatedFmt<'a, T>(pub &'a T, pub usize);
 
-    impl<'a, T> Debug for TruncatedFmt<'a, T>
+    impl<T> Debug for TruncatedFmt<'_, T>
     where
         T: Debug,
     {
@@ -1226,7 +1226,7 @@ pub mod truncated_fmt {
         }
     }
 
-    impl<'a, T> Display for TruncatedFmt<'a, T>
+    impl<T> Display for TruncatedFmt<'_, T>
     where
         T: Display,
     {
diff --git a/src/utils/pgwire/src/types.rs b/src/utils/pgwire/src/types.rs
index c76aa20aac4cd..e8d7dc52101aa 100644
--- a/src/utils/pgwire/src/types.rs
+++ b/src/utils/pgwire/src/types.rs
@@ -94,7 +94,7 @@ where
     default_format: Format,
 }
 
-impl<'a, 'b> FormatIterator<'a, 'b> {
+impl<'a> FormatIterator<'a, '_> {
     pub fn new(provided_formats: &'a [Format], actual_len: usize) -> Result<Self, String> {
         if !provided_formats.is_empty()
             && provided_formats.len() != 1
diff --git a/src/utils/runtime/Cargo.toml b/src/utils/runtime/Cargo.toml
index 44ad04bd40aab..a16d1e5bb8495 100644
--- a/src/utils/runtime/Cargo.toml
+++ b/src/utils/runtime/Cargo.toml
@@ -26,6 +26,7 @@ pprof = { version = "0.13", features = ["flamegraph"] }
 risingwave_common = { workspace = true }
 risingwave_variables = { workspace = true }
 rlimit = "0.10"
+rustls = "0.23.5"
 # Explicitly specify the tokio version used in RisingWave runtime
 rw-tokio = { version = "=1.38.0", package = "tokio" }
 thiserror-ext = { workspace = true }
diff --git a/src/utils/runtime/src/lib.rs b/src/utils/runtime/src/lib.rs
index a5014bdf9a216..34bd836f0629b 100644
--- a/src/utils/runtime/src/lib.rs
+++ b/src/utils/runtime/src/lib.rs
@@ -74,6 +74,12 @@ where
 {
     set_panic_hook();
 
+    rustls::crypto::aws_lc_rs::default_provider()
+        .install_default()
+        .inspect_err(|e| {
+            tracing::error!(?e, "Failed to install default crypto provider.");
+        })
+        .unwrap();
     risingwave_variables::init_server_start_time();
 
     // `TOKIO` will be read by tokio. Duplicate `RW` for compatibility.